From d5a17eda019297236129aa698fb3ea5c9e11faff Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 19 Apr 2024 14:36:06 +0530 Subject: [PATCH 01/50] Initial commit for RemoteRoutingTableService setup Signed-off-by: Himshikha Gupta --- .../remote/RemoteRoutingTableService.java | 104 ++++++++++++++++++ .../cluster/routing/remote/package-info.java | 10 ++ .../common/settings/ClusterSettings.java | 6 +- .../common/settings/FeatureFlagSettings.java | 3 +- .../opensearch/common/util/FeatureFlags.java | 15 ++- .../remote/RemoteClusterStateService.java | 14 +++ .../remotestore/RemoteStoreNodeAttribute.java | 20 ++++ 7 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java create mode 100644 server/src/main/java/org/opensearch/cluster/routing/remote/package-info.java diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java new file mode 100644 index 0000000000000..7778353a75047 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.remote; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.blobstore.BlobStoreRepository; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.function.Supplier; + +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; + +/** + * A Service which provides APIs to upload and download routing table from remote store. + * + * @opensearch.internal + */ +public class RemoteRoutingTableService implements Closeable { + + /** + * Cluster setting to specify if routing table should be published to remote store + */ + public static final Setting REMOTE_ROUTING_TABLE_ENABLED_SETTING = Setting.boolSetting( + "cluster.remote_store.routing.enabled", + false, + Setting.Property.NodeScope, + Setting.Property.Final + ); + private static final Logger logger = LogManager.getLogger(RemoteRoutingTableService.class); + private final Settings settings; + private final Supplier repositoriesService; + private final ClusterSettings clusterSettings; + private BlobStoreRepository blobStoreRepository; + + public RemoteRoutingTableService(Supplier repositoriesService, + Settings settings, + ClusterSettings clusterSettings) { + assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; + this.repositoriesService = repositoriesService; + this.settings = settings; + this.clusterSettings = clusterSettings; + } + + public List writeFullRoutingTable(ClusterState clusterState, String previousClusterUUID) { + return null; + } + + public List writeIncrementalMetadata( + ClusterState previousClusterState, + ClusterState clusterState, + ClusterMetadataManifest previousManifest) { + return null; + } + + public RoutingTable getLatestRoutingTable(String clusterName, String clusterUUID) { + return null; + } + + public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest previousManifest, String clusterName, String clusterUUID) { + return null; + } + + private void deleteStaleRoutingTable(String clusterName, String clusterUUID, int manifestsToRetain) { + } + + @Override + public void close() throws IOException { + if (blobStoreRepository != null) { + IOUtils.close(blobStoreRepository); + } + } + + public void start() { + assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled"; + final String remoteStoreRepo = settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY + ); + assert remoteStoreRepo != null : "Remote routing table repository is not configured"; + final Repository repository = repositoriesService.get().repository(remoteStoreRepo); + assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; + blobStoreRepository = (BlobStoreRepository) repository; + } + +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/package-info.java b/server/src/main/java/org/opensearch/cluster/routing/remote/package-info.java new file mode 100644 index 0000000000000..9fe016e783f20 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Package containing class to perform operations on remote routing table */ +package org.opensearch.cluster.routing.remote; diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index dab0f6bcf1c85..0b101135a00e2 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -76,6 +76,7 @@ import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterManagerService; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; @@ -732,7 +733,10 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, - RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING + RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, + + // Remote Routing table settings + RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 985eb40711e16..255c1c87f0d89 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -36,6 +36,7 @@ protected FeatureFlagSettings( FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.WRITEABLE_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, - FeatureFlags.PLUGGABLE_CACHE_SETTING + FeatureFlags.PLUGGABLE_CACHE_SETTING, + FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index bdfce72d106d3..abee98470f925 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -67,6 +67,11 @@ public class FeatureFlags { */ public static final String PLUGGABLE_CACHE = "opensearch.experimental.feature.pluggable.caching.enabled"; + /** + * Gates the functionality of remote routing table. + */ + public static final String REMOTE_ROUTING_TABLE_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.routing.enabled"; + public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_STORE_MIGRATION_EXPERIMENTAL, false, @@ -93,6 +98,13 @@ public class FeatureFlags { public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); + public static final Setting REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING = Setting.boolSetting( + REMOTE_ROUTING_TABLE_EXPERIMENTAL, + false, + Property.NodeScope + ); + + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -100,7 +112,8 @@ public class FeatureFlags { TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, WRITEABLE_REMOTE_INDEX_SETTING, - PLUGGABLE_CACHE_SETTING + PLUGGABLE_CACHE_SETTING, + REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING ); /** * Should store the settings from opensearch.yml. diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c892b475d71da..8271b1323e207 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -64,6 +65,7 @@ import java.util.stream.Collectors; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** @@ -162,6 +164,7 @@ public class RemoteClusterStateService implements Closeable { private final ThreadPool threadpool; private BlobStoreRepository blobStoreRepository; private BlobStoreTransferService blobStoreTransferService; + private RemoteRoutingTableService remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; private volatile TimeValue indexMetadataUploadTimeout; @@ -206,6 +209,11 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); + + if(isRemoteRoutingTableEnabled(settings)) { + this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, + settings, clusterSettings); + } } private BlobStoreTransferService getBlobStoreTransferService() { @@ -570,6 +578,9 @@ public void close() throws IOException { if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } + if(this.remoteRoutingTableService != null) { + this.remoteRoutingTableService.close(); + } } public void start() { @@ -581,6 +592,9 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; + if(this.remoteRoutingTableService != null) { + this.remoteRoutingTableService.start(); + } } private ClusterMetadataManifest uploadManifest( diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index a3bfe1195d8cc..f1c2ee72a15e0 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -12,7 +12,9 @@ import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -27,6 +29,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; + /** * This is an abstraction for validating and storing information specific to remote backed storage nodes. * @@ -45,6 +49,8 @@ public class RemoteStoreNodeAttribute { + "." + CryptoMetadata.SETTINGS_KEY; public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; + public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing.repository"; + private final RepositoriesMetadata repositoriesMetadata; /** @@ -151,6 +157,10 @@ private Set getValidatedRepositoryNames(DiscoveryNode node) { } else if (node.getAttributes().containsKey(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); } + if (node.getAttributes().containsKey(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)){ + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + } + return repositoryNames; } @@ -181,6 +191,16 @@ public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { && isRemoteClusterStateAttributePresent(settings); } + public static boolean isRemoteRoutingTableAttributePresent(Settings settings) { + return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) + .isEmpty() == false; + } + + public static boolean isRemoteRoutingTableEnabled(Settings settings) { + return FeatureFlags.isEnabled(REMOTE_ROUTING_TABLE_EXPERIMENTAL) && RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.get(settings) + && isRemoteRoutingTableAttributePresent(settings); + } + public RepositoriesMetadata getRepositoriesMetadata() { return this.repositoriesMetadata; } From 9176015df2852ef2cb7994f3d931f06c8d7abd14 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 22 Apr 2024 16:43:46 +0530 Subject: [PATCH 02/50] Adds unit test for remote routing setup Signed-off-by: Himshikha Gupta --- .../remote/RemoteClusterStateService.java | 5 + .../RemoteRoutingTableServiceTests.java | 92 +++++++++++++++++++ .../RemoteClusterStateServiceTests.java | 34 ++++++- 3 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 8271b1323e207..69214d6b22a61 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -741,6 +741,11 @@ public TimeValue getMetadataManifestUploadTimeout() { return this.metadataManifestUploadTimeout; } + //Package private for unit test + RemoteRoutingTableService getRemoteRoutingTableService() { + return this.remoteRoutingTableService; + } + static String getManifestFileName(long term, long version, boolean committed) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ return String.join( diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java new file mode 100644 index 0000000000000..4026a36e76600 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -0,0 +1,92 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.remote; + +import org.junit.After; +import org.junit.Before; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.repositories.FilterRepository; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.RepositoryMissingException; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.function.Supplier; + +import static org.mockito.Mockito.*; +import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; + +public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { + + private RemoteRoutingTableService remoteRoutingTableService; + private ClusterSettings clusterSettings; + private Supplier repositoriesServiceSupplier; + private RepositoriesService repositoriesService; + private BlobStoreRepository blobStoreRepository; + + @Before + public void setup() { + repositoriesServiceSupplier = mock(Supplier.class); + repositoriesService = mock(RepositoriesService.class); + when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); + + Settings settings = Settings.builder() + .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), true) + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .build(); + + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + + blobStoreRepository = mock(BlobStoreRepository.class); + when(repositoriesService.repository("routing_repository")).thenReturn(blobStoreRepository); + + Settings nodeSettings = Settings.builder().put(REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + + remoteRoutingTableService = new RemoteRoutingTableService( + repositoriesServiceSupplier, + settings, + clusterSettings + ); + } + + @After + public void teardown() throws Exception { + super.tearDown(); + remoteRoutingTableService.close(); + } + + + public void testFailInitializationWhenRemoteRoutingDisabled() { + final Settings settings = Settings.builder().build(); + assertThrows( + AssertionError.class, + () -> new RemoteRoutingTableService( + repositoriesServiceSupplier, + settings, + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ) + ); + } + + public void testFailStartWhenRepositoryNotSet() { + doThrow(new RepositoryMissingException("repository missing")).when(repositoriesService).repository("routing_repository"); + assertThrows(RepositoryMissingException.class, () -> remoteRoutingTableService.start()); + } + + public void testFailStartWhenNotBlobRepository() { + final FilterRepository filterRepository = mock(FilterRepository.class); + when(repositoriesService.repository("routing_repository")).thenReturn(filterRepository); + assertThrows(AssertionError.class, () -> remoteRoutingTableService.start()); + } + +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 65477051cdb30..78aa1173bea62 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -31,6 +32,7 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; @@ -77,6 +79,7 @@ import org.mockito.ArgumentMatchers; import static java.util.stream.Collectors.toList; +import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; @@ -84,9 +87,6 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -99,6 +99,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.*; public class RemoteClusterStateServiceTests extends OpenSearchTestCase { @@ -1218,6 +1219,33 @@ public void testGlobalMetadataUploadWaitTimeSetting() { assertEquals(globalMetadataUploadTimeout, remoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); } + public void testRemoteRoutingTableNotInitializedWhenDisabled() { + assertNull(remoteClusterStateService.getRemoteRoutingTableService()); + } + + public void testRemoteRoutingTableInitializedWhenEnabled() { + Settings newSettings = Settings.builder() + .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), true) + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + clusterSettings.applySettings(newSettings); + + Settings nodeSettings = Settings.builder().put(REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + + remoteClusterStateService = new RemoteClusterStateService( + "test-node-id", + repositoriesServiceSupplier, + newSettings, + clusterSettings, + () -> 0L, + threadPool + ); + assertNotNull(remoteClusterStateService.getRemoteRoutingTableService()); + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } From 7d9badde496038c10955bbffd6f0b010123e6db1 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 7 May 2024 14:22:20 +0530 Subject: [PATCH 03/50] Updating remote routing table setting name Signed-off-by: Himshikha Gupta --- .../cluster/routing/remote/RemoteRoutingTableService.java | 2 +- .../src/main/java/org/opensearch/common/util/FeatureFlags.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 7778353a75047..7755fb5b74e4e 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -41,7 +41,7 @@ public class RemoteRoutingTableService implements Closeable { * Cluster setting to specify if routing table should be published to remote store */ public static final Setting REMOTE_ROUTING_TABLE_ENABLED_SETTING = Setting.boolSetting( - "cluster.remote_store.routing.enabled", + "cluster.remote_store.routing_table.enabled", false, Setting.Property.NodeScope, Setting.Property.Final diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index abee98470f925..acce722919d90 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -70,7 +70,7 @@ public class FeatureFlags { /** * Gates the functionality of remote routing table. */ - public static final String REMOTE_ROUTING_TABLE_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.routing.enabled"; + public static final String REMOTE_ROUTING_TABLE_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.routing_table.enabled"; public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_STORE_MIGRATION_EXPERIMENTAL, From ad7ecf5d376254756ca3d7fb61c925dac2f7cd22 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 7 May 2024 14:28:20 +0530 Subject: [PATCH 04/50] Update remote routing table repo setting name Signed-off-by: Himshikha Gupta --- .../opensearch/node/remotestore/RemoteStoreNodeAttribute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index f1c2ee72a15e0..542f8143a3af6 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -49,7 +49,7 @@ public class RemoteStoreNodeAttribute { + "." + CryptoMetadata.SETTINGS_KEY; public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; - public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing.repository"; + public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing_table.repository"; private final RepositoriesMetadata repositoriesMetadata; From a66aaafac02aebed7659b42dc0fcd69d64306b02 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Wed, 17 Apr 2024 14:07:08 +0530 Subject: [PATCH 05/50] Initial commit for index routing table manifest Signed-off-by: Bukhtawar Khan --- .../remote/ClusterMetadataManifest.java | 98 ++++++++++++++++++- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4725f40076ce2..4e5891d154de0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -35,6 +35,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. + public static final int CODEC_V2 = 2; // In Codec V2 we introduce index routing-metadata in manifest file. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -48,6 +49,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); + private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); private static long term(Object[] fields) { return (long) fields[0]; @@ -97,6 +99,10 @@ private static String globalMetadataFileName(Object[] fields) { return (String) fields[11]; } + private static List indicesRouting(Object[] fields) { + return (List) fields[12]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> new ClusterMetadataManifest( @@ -133,11 +139,31 @@ private static String globalMetadataFileName(Object[] fields) { ) ); - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V1; + private static final ConstructingObjectParser PARSER_V2 = new ConstructingObjectParser<>( + "cluster_metadata_manifest", + fields -> new ClusterMetadataManifest( + term(fields), + version(fields), + clusterUUID(fields), + stateUUID(fields), + opensearchVersion(fields), + nodeId(fields), + committed(fields), + codecVersion(fields), + globalMetadataFileName(fields), + indices(fields), + previousClusterUUID(fields), + clusterUUIDCommitted(fields), + indicesRouting(fields) + ) + ); + + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); + declareParser(PARSER_V2, CODEC_V2); } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -160,6 +186,13 @@ private static void declareParser(ConstructingObjectParser= CODEC_V2) { + parser.declareObjectArray( + ConstructingObjectParser.constructorArg(), + (p, c) -> UploadedIndexMetadata.fromXContent(p), + INDICES_ROUTING_FIELD + ); + } } private final int codecVersion; @@ -174,6 +207,7 @@ private static void declareParser(ConstructingObjectParser indicesRouting; public List getIndices() { return indices; @@ -223,6 +257,10 @@ public String getGlobalMetadataFileName() { return globalMetadataFileName; } + public List getIndicesRouting() { + return indicesRouting; + } + public ClusterMetadataManifest( long clusterTerm, long version, @@ -237,6 +275,25 @@ public ClusterMetadataManifest( String previousClusterUUID, boolean clusterUUIDCommitted ) { + this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, null); + } + + public ClusterMetadataManifest( + long clusterTerm, + long version, + String clusterUUID, + String stateUUID, + Version opensearchVersion, + String nodeId, + boolean committed, + int codecVersion, + String globalMetadataFileName, + List indices, + String previousClusterUUID, + boolean clusterUUIDCommitted, + List indicesRouting + ) { this.clusterTerm = clusterTerm; this.stateVersion = version; this.clusterUUID = clusterUUID; @@ -249,6 +306,7 @@ public ClusterMetadataManifest( this.indices = Collections.unmodifiableList(indices); this.previousClusterUUID = previousClusterUUID; this.clusterUUIDCommitted = clusterUUIDCommitted; + this.indicesRouting = Collections.unmodifiableList(indicesRouting); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -262,12 +320,18 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_2_12_0)) { + if (in.getVersion().onOrAfter(Version.V_2_14_0)) { + this.codecVersion = in.readInt(); + this.globalMetadataFileName = in.readString(); + this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); + } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.indicesRouting = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; + this.indicesRouting = null; } } @@ -301,6 +365,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } + if (onOrAfterCodecVersion(CODEC_V2)) { + builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); + { + for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { + uploadedIndexMetadata.toXContent(builder, params); + } + } + } return builder; } @@ -319,6 +391,8 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); + } else if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + out.writeCollection(indicesRouting); } } @@ -342,7 +416,8 @@ public boolean equals(Object o) { && Objects.equals(previousClusterUUID, that.previousClusterUUID) && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) - && Objects.equals(codecVersion, that.codecVersion); + && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(indicesRouting, that.indicesRouting); } @Override @@ -359,7 +434,8 @@ public int hashCode() { nodeId, committed, previousClusterUUID, - clusterUUIDCommitted + clusterUUIDCommitted, + indicesRouting ); } @@ -399,12 +475,18 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; + private List indicesRouting; public Builder indices(List indices) { this.indices = indices; return this; } + public Builder indicesRouting(List indicesRouting) { + this.indicesRouting = indicesRouting; + return this; + } + public Builder codecVersion(int codecVersion) { this.codecVersion = codecVersion; return this; @@ -454,6 +536,10 @@ public List getIndices() { return indices; } + public List getIndicesRouting() { + return indicesRouting; + } + public Builder previousClusterUUID(String previousClusterUUID) { this.previousClusterUUID = previousClusterUUID; return this; @@ -481,6 +567,7 @@ public Builder(ClusterMetadataManifest manifest) { this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; + this.indicesRouting = new ArrayList<>(manifest.indicesRouting); } public ClusterMetadataManifest build() { @@ -496,7 +583,8 @@ public ClusterMetadataManifest build() { globalMetadataFileName, indices, previousClusterUUID, - clusterUUIDCommitted + clusterUUIDCommitted, + indicesRouting ); } From ad480ee208a8637920cba04d85c8fb9c426aa96c Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 20 Apr 2024 00:00:39 +0530 Subject: [PATCH 06/50] Changes for IndexRoutingTableHeader Signed-off-by: Bukhtawar Khan --- .../stream}/BufferedChecksumStreamInput.java | 2 +- .../stream}/BufferedChecksumStreamOutput.java | 2 +- .../org/opensearch/common/util/BigArrays.java | 2 +- .../routingtable/IndexRoutingTableHeader.java | 135 ++++++++++++++++++ .../index/translog/BaseTranslogReader.java | 1 + .../opensearch/index/translog/Translog.java | 2 + .../index/translog/TranslogHeader.java | 2 + .../index/translog/TranslogSnapshot.java | 1 + .../index/translog/TranslogWriter.java | 1 + .../coordination/CoordinationStateTests.java | 2 +- .../remote/ClusterMetadataManifestTests.java | 6 +- .../snapshots/BlobStoreFormatTests.java | 2 +- 12 files changed, 150 insertions(+), 8 deletions(-) rename server/src/main/java/org/opensearch/{index/translog => common/io/stream}/BufferedChecksumStreamInput.java (99%) rename server/src/main/java/org/opensearch/{index/translog => common/io/stream}/BufferedChecksumStreamOutput.java (98%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java similarity index 99% rename from server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java rename to server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java index f75f27b7bcb91..f3341712275f9 100644 --- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.index.translog; +package org.opensearch.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.apache.lucene.util.BitUtil; diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java similarity index 98% rename from server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java rename to server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java index 9e96664c79cc5..254f228f1c739 100644 --- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.index.translog; +package org.opensearch.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java index 92371c2c77ef9..734e5535c3cf4 100644 --- a/server/src/main/java/org/opensearch/common/util/BigArrays.java +++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java @@ -49,7 +49,7 @@ import java.util.Arrays; /** - * Utility class to work with arrays. + * Utility class to work with arrays.Ø * * @opensearch.api * */ diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java new file mode 100644 index 0000000000000..87d94dc1147d1 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -0,0 +1,135 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.store.InputStreamDataInput; +import org.apache.lucene.store.OutputStreamDataOutput; +import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; + +import java.io.EOFException; +import java.io.IOException; + +/** + * The stored header information for the individual index routing table + */ +public class IndexRoutingTableHeader { + + private int routingTableVersion; + + private String indexName; + + private Version nodeVersion; + + public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; + + public static final int INITIAL_VERSION = 1; + + public static final int CURRENT_VERSION = INITIAL_VERSION; + + + public IndexRoutingTableHeader(int routingTableVersion, String indexName, Version nodeVersion) { + this.routingTableVersion = routingTableVersion; + this.indexName = indexName; + this.nodeVersion = nodeVersion; + } + + /** + * Returns the bytes reference for the {@link IndexRoutingTableHeader} + * @return the {@link BytesReference} + * @throws IOException + */ + public BytesReference write() throws IOException { + BytesReference bytesReference; + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput)) { + CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); + // Write version + out.writeInt(routingTableVersion); + out.writeInt(nodeVersion.id); + out.writeString(indexName); + // Checksum header + out.writeInt((int) out.getChecksum()); + out.flush(); + bytesReference = bytesStreamOutput.bytes(); + } + return bytesReference; + } + + + /** + * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} + * @param inBytes + * @param source + * @return + * @throws IOException + */ + public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { + try { + try(BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { + readHeaderVersion(in); + final int version = in.readInt(); + final int nodeVersion = in.readInt(); + final String name = in.readString(); + verifyChecksum(in); + assert version >= 0 : "Version must be non-negative [" + version + "]"; + assert in.readByte() == -1 : "Header is not fully read"; + return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); + } + } catch (EOFException e) { + throw new IOException("index routing header truncated", e); + } + } + + + static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { + // This absolutely must come first, or else reading the checksum becomes part of the checksum + long expectedChecksum = in.getChecksum(); + long readChecksum = Integer.toUnsignedLong(in.readInt()); + if (readChecksum != expectedChecksum) { + throw new IOException( + "checksum verification failed - expected: 0x" + + Long.toHexString(expectedChecksum) + + ", got: 0x" + + Long.toHexString(readChecksum) + ); + } + } + + static int readHeaderVersion(final StreamInput in) throws IOException { + final int version; + try { + version = CodecUtil.checkHeader(new InputStreamDataInput(in), INDEX_ROUTING_HEADER_CODEC, INITIAL_VERSION, CURRENT_VERSION); + } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { + throw new IOException("index routing table header corrupted", e); + } + return version; + } + + public int getRoutingTableVersion() { + return routingTableVersion; + } + + public String getIndexName() { + return indexName; + } + + public Version getNodeVersion() { + return nodeVersion; + } +} diff --git a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java index d6fa2a2e53de3..9088eb6b20fb8 100644 --- a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java +++ b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java @@ -32,6 +32,7 @@ package org.opensearch.index.translog; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 842e9c77d2350..18b10cb886996 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -38,6 +38,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index 7b5be9505f27a..c622214a3a69d 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -40,6 +40,8 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.Channels; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java index 89718156cbbe8..a6e322bf58fff 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java @@ -32,6 +32,7 @@ package org.opensearch.index.translog; import org.opensearch.common.io.Channels; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.index.seqno.SequenceNumbers; import java.io.EOFException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index 86f7567f3333d..da3c7a8dee219 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -42,6 +42,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Channels; import org.opensearch.common.io.DiskIoBufferPool; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 1c0dc7fc1ca2d..0fe52d2e72d4b 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -942,7 +942,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep randomAlphaOfLength(10), Collections.emptyList(), randomAlphaOfLength(10), - true + true, ); Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 6c9a3201656d7..647f4c913931f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -40,7 +40,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true + true, ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -67,7 +67,7 @@ public void testClusterMetadataManifestXContent() throws IOException { "test-global-metadata-file", Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true + true, ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -93,7 +93,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "test-global-metadata-file", randomUploadedIndexMetadataList(), "yfObdx8KSMKKrXf8UyHhM", - true + true, ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index c5f36fcc01983..8c64f9a3170b2 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -55,7 +55,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.translog.BufferedChecksumStreamOutput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.test.OpenSearchTestCase; From 2ad70c4fb9df43d99ce3714df1b4303f7589dc9e Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 20 Apr 2024 00:04:24 +0530 Subject: [PATCH 07/50] Revert unintentional changes for IndexRoutingTableHeader Signed-off-by: Bukhtawar Khan --- .../src/main/java/org/opensearch/common/util/BigArrays.java | 2 +- .../cluster/coordination/CoordinationStateTests.java | 2 +- .../gateway/remote/ClusterMetadataManifestTests.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java index 734e5535c3cf4..92371c2c77ef9 100644 --- a/server/src/main/java/org/opensearch/common/util/BigArrays.java +++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java @@ -49,7 +49,7 @@ import java.util.Arrays; /** - * Utility class to work with arrays.Ø + * Utility class to work with arrays. * * @opensearch.api * */ diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 0fe52d2e72d4b..1c0dc7fc1ca2d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -942,7 +942,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep randomAlphaOfLength(10), Collections.emptyList(), randomAlphaOfLength(10), - true, + true ); Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 647f4c913931f..6c9a3201656d7 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -40,7 +40,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true, + true ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -67,7 +67,7 @@ public void testClusterMetadataManifestXContent() throws IOException { "test-global-metadata-file", Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true, + true ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -93,7 +93,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "test-global-metadata-file", randomUploadedIndexMetadataList(), "yfObdx8KSMKKrXf8UyHhM", - true, + true ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( From acc172ea899fe2e0c5a75fe1a030b081af14aaca Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 20 Apr 2024 00:04:24 +0530 Subject: [PATCH 08/50] Revert unintentional changes for IndexRoutingTableHeader Signed-off-by: Bukhtawar Khan From f65b102996fd23e405ce4ea3bdfcdc54111cfe21 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sun, 21 Apr 2024 00:15:47 +0530 Subject: [PATCH 09/50] Changes for IndexRoutingTableInputStream Signed-off-by: Bukhtawar Khan --- .../routingtable/IndexRoutingTableHeader.java | 27 ++- .../IndexRoutingTableInputStream.java | 157 ++++++++++++++++++ 2 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 87d94dc1147d1..04a0f1868b64f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -15,12 +15,12 @@ import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; import org.opensearch.Version; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import java.io.EOFException; import java.io.IOException; @@ -30,11 +30,11 @@ */ public class IndexRoutingTableHeader { - private int routingTableVersion; + private final long routingTableVersion; - private String indexName; + private final String indexName; - private Version nodeVersion; + private final Version nodeVersion; public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; @@ -42,8 +42,7 @@ public class IndexRoutingTableHeader { public static final int CURRENT_VERSION = INITIAL_VERSION; - - public IndexRoutingTableHeader(int routingTableVersion, String indexName, Version nodeVersion) { + public IndexRoutingTableHeader(long routingTableVersion, String indexName, Version nodeVersion) { this.routingTableVersion = routingTableVersion; this.indexName = indexName; this.nodeVersion = nodeVersion; @@ -56,11 +55,13 @@ public IndexRoutingTableHeader(int routingTableVersion, String indexName, Versio */ public BytesReference write() throws IOException { BytesReference bytesReference; - try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput)) { + try ( + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) + ) { CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); // Write version - out.writeInt(routingTableVersion); + out.writeLong(routingTableVersion); out.writeInt(nodeVersion.id); out.writeString(indexName); // Checksum header @@ -71,7 +72,6 @@ public BytesReference write() throws IOException { return bytesReference; } - /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} * @param inBytes @@ -81,7 +81,7 @@ public BytesReference write() throws IOException { */ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { try { - try(BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { + try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); final int version = in.readInt(); final int nodeVersion = in.readInt(); @@ -96,7 +96,6 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce } } - static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { // This absolutely must come first, or else reading the checksum becomes part of the checksum long expectedChecksum = in.getChecksum(); @@ -121,7 +120,7 @@ static int readHeaderVersion(final StreamInput in) throws IOException { return version; } - public int getRoutingTableVersion() { + public long getRoutingTableVersion() { return routingTableVersion; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java new file mode 100644 index 0000000000000..ac65232e9a24d --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.opensearch.Version; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; + +public class IndexRoutingTableInputStream extends InputStream { + + /** + * The buffer where data is stored. + */ + protected byte[] buf; + + /** + * The number of valid bytes in the buffer. + */ + protected int count; + + /** + * The buffer left over from the last fill + */ + protected byte[] leftOverBuf; + + /** + * The mark position + */ + protected int markPos = -1; + + /** + * The read limit + */ + protected int markLimit; + + /** + * The position + */ + protected int pos; + + private static final int BUFFER_SIZE = 8192; + + private final IndexRoutingTableHeader indexRoutingTableHeader; + + private final Iterator shardIter; + + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { + this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); + } + + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion, int size) + throws IOException { + this.buf = new byte[size]; + this.shardIter = indexRoutingTable.iterator(); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); + initialFill(); + } + + @Override + public int read() throws IOException { + if (pos >= count) { + maybeResizeAndFill(); + if (pos >= count) return -1; + } + return buf[pos++] & 0xff; + } + + private void initialFill() throws IOException { + BytesReference bytesReference = indexRoutingTableHeader.write(); + buf = bytesReference.toBytesRef().bytes; + count = bytesReference.length(); + fill(buf); + } + + private void fill(byte[] buf) throws IOException { + if (leftOverBuf != null) { + System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + } + if (count < buf.length && shardIter.hasNext()) { + IndexShardRoutingTable next = shardIter.next(); + BytesReference bytesRef; + try ( + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) + ) { + IndexShardRoutingTable.Builder.writeTo(next, out); + // Checksum header + out.writeInt((int) out.getChecksum()); + out.flush(); + bytesRef = bytesStreamOutput.bytes(); + } + if (bytesRef.length() < buf.length - count) { + System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, bytesRef.length()); + count += bytesRef.length(); + leftOverBuf = null; + } else { + System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); + count += buf.length - count; + leftOverBuf = new byte[bytesRef.length() - count]; + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count + 1, leftOverBuf, 0, bytesRef.length() - count); + } + } + } + + private void maybeResizeAndFill() throws IOException { + byte[] buffer = buf; + if (markPos == -1) pos = 0; /* no mark: throw away the buffer */ + else if (pos >= buffer.length) { /* no room left in buffer */ + if (markPos > 0) { /* can throw away early part of the buffer */ + int sz = pos - markPos; + System.arraycopy(buffer, markPos, buffer, 0, sz); + pos = sz; + markPos = 0; + } else if (buffer.length >= markLimit) { + markPos = -1; /* buffer got too big, invalidate mark */ + pos = 0; /* drop buffer contents */ + } else { /* grow buffer */ + int nsz = markLimit + 1; + byte[] nbuf = new byte[nsz]; + System.arraycopy(buffer, 0, nbuf, 0, pos); + buffer = nbuf; + } + } + count = pos; + fill(buffer); + } + + @Override + public void mark(int readlimit) { + markLimit = readlimit; + markPos = pos; + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void reset() throws IOException { + if (markPos < 0) throw new IOException("Resetting to invalid mark"); + pos = markPos; + } +} From 441f52023d3757e1073fefb629fe145f19ff671f Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 7 May 2024 16:11:06 +0530 Subject: [PATCH 10/50] Fixing IndexRoutingTableInputStream and moving checksum to end to file Signed-off-by: Himshikha Gupta --- .../remote/ClusterMetadataManifest.java | 15 +++-- .../routingtable/IndexRoutingTableHeader.java | 32 ++--------- .../IndexRoutingTableInputStream.java | 57 ++++++++++++------- .../IndexRoutingTableHeaderTests.java | 35 ++++++++++++ 4 files changed, 86 insertions(+), 53 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4e5891d154de0..0279f8e0fd805 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -276,7 +276,7 @@ public ClusterMetadataManifest( boolean clusterUUIDCommitted ) { this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, - globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, null); + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, new ArrayList<>()); } public ClusterMetadataManifest( @@ -355,7 +355,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startArray(INDICES_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indices) { + builder.startObject(); uploadedIndexMetadata.toXContent(builder, params); + builder.endObject(); } } builder.endArray(); @@ -369,9 +371,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { + builder.startObject(); uploadedIndexMetadata.toXContent(builder, params); + builder.endObject(); } } + builder.endArray(); } return builder; } @@ -391,7 +396,8 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); - } else if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + } + if (out.getVersion().onOrAfter(Version.V_2_14_0)) { out.writeCollection(indicesRouting); } } @@ -659,11 +665,10 @@ public String getIndexUUID() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject() + return builder .field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) .field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID()) - .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()) - .endObject(); + .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 04a0f1868b64f..23ab700d5a34f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -21,6 +21,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.EOFException; import java.io.IOException; @@ -50,26 +51,16 @@ public IndexRoutingTableHeader(long routingTableVersion, String indexName, Versi /** * Returns the bytes reference for the {@link IndexRoutingTableHeader} - * @return the {@link BytesReference} * @throws IOException */ - public BytesReference write() throws IOException { - BytesReference bytesReference; - try ( - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) - ) { + public void write(StreamOutput out) throws IOException { CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); // Write version out.writeLong(routingTableVersion); out.writeInt(nodeVersion.id); out.writeString(indexName); - // Checksum header - out.writeInt((int) out.getChecksum()); + out.flush(); - bytesReference = bytesStreamOutput.bytes(); - } - return bytesReference; } /** @@ -83,10 +74,9 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); - final int version = in.readInt(); + final long version = in.readLong(); final int nodeVersion = in.readInt(); final String name = in.readString(); - verifyChecksum(in); assert version >= 0 : "Version must be non-negative [" + version + "]"; assert in.readByte() == -1 : "Header is not fully read"; return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); @@ -96,20 +86,6 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce } } - static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { - // This absolutely must come first, or else reading the checksum becomes part of the checksum - long expectedChecksum = in.getChecksum(); - long readChecksum = Integer.toUnsignedLong(in.readInt()); - if (readChecksum != expectedChecksum) { - throw new IOException( - "checksum verification failed - expected: 0x" - + Long.toHexString(expectedChecksum) - + ", got: 0x" - + Long.toHexString(readChecksum) - ); - } - } - static int readHeaderVersion(final StreamInput in) throws IOException { final int version; try { diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index ac65232e9a24d..40e5908d5c65c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -54,8 +54,9 @@ public class IndexRoutingTableInputStream extends InputStream { private static final int BUFFER_SIZE = 8192; private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; + private final BytesStreamOutput bytesStreamOutput; + private final BufferedChecksumStreamOutput out; public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); @@ -66,7 +67,10 @@ public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long ve this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); - initialFill(); + this.bytesStreamOutput = new BytesStreamOutput(); + this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); + + initialFill(indexRoutingTable.shards().size()); } @Override @@ -78,39 +82,52 @@ public int read() throws IOException { return buf[pos++] & 0xff; } - private void initialFill() throws IOException { - BytesReference bytesReference = indexRoutingTableHeader.write(); - buf = bytesReference.toBytesRef().bytes; - count = bytesReference.length(); + private void initialFill(int shardCount) throws IOException { + indexRoutingTableHeader.write(out); + out.writeVInt(shardCount); + + System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0 , buf, 0, bytesStreamOutput.bytes().length()); + count = bytesStreamOutput.bytes().length(); + bytesStreamOutput.reset(); fill(buf); } private void fill(byte[] buf) throws IOException { if (leftOverBuf != null) { - System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + if(leftOverBuf.length > buf.length - count) { + // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in leftOverBuf. + System.arraycopy(leftOverBuf, 0, buf, count, buf.length - count); + byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; + System.arraycopy(leftOverBuf, buf.length - count , tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); + leftOverBuf = tempLeftOverBuffer; + count = buf.length - count; + } else { + System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + count += leftOverBuf.length; + leftOverBuf = null; + } } + if (count < buf.length && shardIter.hasNext()) { IndexShardRoutingTable next = shardIter.next(); - BytesReference bytesRef; - try ( - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) - ) { - IndexShardRoutingTable.Builder.writeTo(next, out); - // Checksum header - out.writeInt((int) out.getChecksum()); - out.flush(); - bytesRef = bytesStreamOutput.bytes(); + IndexShardRoutingTable.Builder.writeTo(next, out); + //Add checksum for the file after all shards are done + if(!shardIter.hasNext()) { + out.writeLong(out.getChecksum()); } + out.flush(); + BytesReference bytesRef = bytesStreamOutput.bytes(); + bytesStreamOutput.reset(); + if (bytesRef.length() < buf.length - count) { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, bytesRef.length()); count += bytesRef.length(); leftOverBuf = null; } else { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); - count += buf.length - count; - leftOverBuf = new byte[bytesRef.length() - count]; - System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count + 1, leftOverBuf, 0, bytesRef.length() - count); + leftOverBuf = new byte[bytesRef.length() - (buf.length - count)]; + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count , leftOverBuf, 0, bytesRef.length() - (buf.length - count)); + count = buf.length; } } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java new file mode 100644 index 0000000000000..068db554b4226 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.InputStreamDataInput; +import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { + + public void testWrite() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + IndexRoutingTableHeader header = new IndexRoutingTableHeader(1, "dummyIndex", Version.V_3_0_0); + header.write(out); + + BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); + CodecUtil.checkHeader(new InputStreamDataInput(in),IndexRoutingTableHeader.INDEX_ROUTING_HEADER_CODEC, IndexRoutingTableHeader.INITIAL_VERSION, IndexRoutingTableHeader.CURRENT_VERSION ); + assertEquals(1, in.readLong()); + assertEquals(Version.V_3_0_0.id, in.readInt()); + assertEquals("dummyIndex", in.readString()); + } + +} From ee70dca9af636bdafae507fe5b004a7de4708f20 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Wed, 8 May 2024 11:09:54 +0530 Subject: [PATCH 11/50] Add read flow for IndexRoutingTable Signed-off-by: Arpit Bandejiya --- .../routingtable/IndexRoutingTableHeader.java | 10 +-- .../IndexRoutingTableInputStreamReader.java | 77 +++++++++++++++++++ .../IndexRoutingTableInputStreamTests.java | 61 +++++++++++++++ 3 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 23ab700d5a34f..e29ce5a79dc02 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -65,22 +65,18 @@ public void write(StreamOutput out) throws IOException { /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} - * @param inBytes - * @param source - * @return + * @param in + * @return IndexRoutingTableHeader * @throws IOException */ - public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { + public static IndexRoutingTableHeader read(BufferedChecksumStreamInput in) throws IOException { try { - try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); final long version = in.readLong(); final int nodeVersion = in.readInt(); final String name = in.readString(); assert version >= 0 : "Version must be non-negative [" + version + "]"; - assert in.readByte() == -1 : "Header is not fully read"; return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); - } } catch (EOFException e) { throw new IOException("index routing header truncated", e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java new file mode 100644 index 0000000000000..35ae9f287d7f2 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IndexRoutingTableInputStreamReader { + + private final StreamInput streamInput; + + private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStreamReader.class); + + public IndexRoutingTableInputStreamReader(InputStream inputStream) throws IOException { + this.streamInput = new InputStreamStreamInput(inputStream); + } + + public Map read() throws IOException { + try { + try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(streamInput, "assertion")) { + // Read the Table Header first + IndexRoutingTableHeader.read(in); + int shards = in.readVInt(); + logger.info("Number of Index Routing Table {}", shards); + Map indicesRouting = new HashMap(Collections.EMPTY_MAP); + for(int i=0; i { + try { + logger.info("IndexShardRoutingTables: {}", indexShardRoutingTables); + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables, + initialRoutingTable.version(), Version.CURRENT); + + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + Map indexShardRoutingTableMap = reader.read(); + + logger.info("indexShardRoutingTableMap: {}", indexShardRoutingTableMap); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + +} From 7aeecc84592d3948b932c5973cf218091297237f Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 13 May 2024 18:08:24 +0530 Subject: [PATCH 12/50] Moving routing table version from IndexRouting stream to manifest Signed-off-by: Himshikha Gupta --- .../remote/ClusterMetadataManifest.java | 33 ++++++++++- .../routingtable/IndexRoutingTableHeader.java | 59 +++++-------------- .../IndexRoutingTableInputStream.java | 9 ++- .../remote/ClusterMetadataManifestTests.java | 29 +++++++++ .../IndexRoutingTableHeaderTests.java | 14 ++--- .../IndexRoutingTableInputStreamTests.java | 12 ++-- 6 files changed, 90 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 0279f8e0fd805..5b7192c624fed 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -49,6 +49,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); + private static final ParseField ROUTING_TABLE_VERSION_FIELD = new ParseField("routing_table_version"); private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); private static long term(Object[] fields) { @@ -99,8 +100,12 @@ private static String globalMetadataFileName(Object[] fields) { return (String) fields[11]; } + private static long routingTableVersion(Object[] fields) { + return (long) fields[12]; + } + private static List indicesRouting(Object[] fields) { - return (List) fields[12]; + return (List) fields[13]; } private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( @@ -154,6 +159,7 @@ private static List indicesRouting(Object[] fields) { indices(fields), previousClusterUUID(fields), clusterUUIDCommitted(fields), + routingTableVersion(fields), indicesRouting(fields) ) ); @@ -187,6 +193,7 @@ private static void declareParser(ConstructingObjectParser= CODEC_V2) { + parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD); parser.declareObjectArray( ConstructingObjectParser.constructorArg(), (p, c) -> UploadedIndexMetadata.fromXContent(p), @@ -207,6 +214,7 @@ private static void declareParser(ConstructingObjectParser indicesRouting; public List getIndices() { @@ -257,6 +265,10 @@ public String getGlobalMetadataFileName() { return globalMetadataFileName; } + public long getRoutingTableVersion() { + return routingTableVersion; + } + public List getIndicesRouting() { return indicesRouting; } @@ -276,7 +288,7 @@ public ClusterMetadataManifest( boolean clusterUUIDCommitted ) { this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, - globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, new ArrayList<>()); + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, -1, new ArrayList<>()); } public ClusterMetadataManifest( @@ -292,6 +304,7 @@ public ClusterMetadataManifest( List indices, String previousClusterUUID, boolean clusterUUIDCommitted, + long routingTableVersion, List indicesRouting ) { this.clusterTerm = clusterTerm; @@ -306,6 +319,7 @@ public ClusterMetadataManifest( this.indices = Collections.unmodifiableList(indices); this.previousClusterUUID = previousClusterUUID; this.clusterUUIDCommitted = clusterUUIDCommitted; + this.routingTableVersion = routingTableVersion; this.indicesRouting = Collections.unmodifiableList(indicesRouting); } @@ -323,14 +337,17 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_14_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.routingTableVersion = in.readLong(); this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.routingTableVersion = -1; this.indicesRouting = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; + this.routingTableVersion = -1; this.indicesRouting = null; } } @@ -368,6 +385,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } if (onOrAfterCodecVersion(CODEC_V2)) { + builder.field(ROUTING_TABLE_VERSION_FIELD.getPreferredName(), getRoutingTableVersion()); builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { @@ -398,6 +416,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(globalMetadataFileName); } if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + out.writeLong(routingTableVersion); out.writeCollection(indicesRouting); } } @@ -423,6 +442,7 @@ public boolean equals(Object o) { && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(routingTableVersion, that.routingTableVersion) && Objects.equals(indicesRouting, that.indicesRouting); } @@ -441,6 +461,7 @@ public int hashCode() { committed, previousClusterUUID, clusterUUIDCommitted, + routingTableVersion, indicesRouting ); } @@ -481,6 +502,7 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; + private long routingTableVersion; private List indicesRouting; public Builder indices(List indices) { @@ -488,6 +510,11 @@ public Builder indices(List indices) { return this; } + public Builder routingTableVersion(long routingTableVersion) { + this.routingTableVersion = routingTableVersion; + return this; + } + public Builder indicesRouting(List indicesRouting) { this.indicesRouting = indicesRouting; return this; @@ -573,6 +600,7 @@ public Builder(ClusterMetadataManifest manifest) { this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; + this.routingTableVersion = manifest.routingTableVersion; this.indicesRouting = new ArrayList<>(manifest.indicesRouting); } @@ -590,6 +618,7 @@ public ClusterMetadataManifest build() { indices, previousClusterUUID, clusterUUIDCommitted, + routingTableVersion, indicesRouting ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index e29ce5a79dc02..bc99fea9b5c09 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -14,12 +14,6 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; -import org.opensearch.Version; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -31,52 +25,27 @@ */ public class IndexRoutingTableHeader { - private final long routingTableVersion; - - private final String indexName; - - private final Version nodeVersion; - public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; - public static final int INITIAL_VERSION = 1; - public static final int CURRENT_VERSION = INITIAL_VERSION; + private final String indexName; - public IndexRoutingTableHeader(long routingTableVersion, String indexName, Version nodeVersion) { - this.routingTableVersion = routingTableVersion; + public IndexRoutingTableHeader(String indexName) { this.indexName = indexName; - this.nodeVersion = nodeVersion; - } - - /** - * Returns the bytes reference for the {@link IndexRoutingTableHeader} - * @throws IOException - */ - public void write(StreamOutput out) throws IOException { - CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); - // Write version - out.writeLong(routingTableVersion); - out.writeInt(nodeVersion.id); - out.writeString(indexName); - - out.flush(); } /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} + * * @param in * @return IndexRoutingTableHeader * @throws IOException */ - public static IndexRoutingTableHeader read(BufferedChecksumStreamInput in) throws IOException { + public static IndexRoutingTableHeader read(StreamInput in) throws IOException { try { - readHeaderVersion(in); - final long version = in.readLong(); - final int nodeVersion = in.readInt(); - final String name = in.readString(); - assert version >= 0 : "Version must be non-negative [" + version + "]"; - return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); + readHeaderVersion(in); + final String name = in.readString(); + return new IndexRoutingTableHeader(name); } catch (EOFException e) { throw new IOException("index routing header truncated", e); } @@ -92,15 +61,19 @@ static int readHeaderVersion(final StreamInput in) throws IOException { return version; } - public long getRoutingTableVersion() { - return routingTableVersion; + /** + * Returns the bytes reference for the {@link IndexRoutingTableHeader} + * + * @throws IOException + */ + public void write(StreamOutput out) throws IOException { + CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); + out.writeString(indexName); + out.flush(); } public String getIndexName() { return indexName; } - public Version getNodeVersion() { - return nodeVersion; - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index 40e5908d5c65c..5332a8a87c9a4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -8,7 +8,6 @@ package org.opensearch.gateway.remote.routingtable; -import org.opensearch.Version; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; @@ -58,15 +57,15 @@ public class IndexRoutingTableInputStream extends InputStream { private final BytesStreamOutput bytesStreamOutput; private final BufferedChecksumStreamOutput out; - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { - this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable) throws IOException { + this(indexRoutingTable, BUFFER_SIZE); } - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion, int size) + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) throws IOException { this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); this.bytesStreamOutput = new BytesStreamOutput(); this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 6c9a3201656d7..73933141ddf23 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -233,6 +233,35 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { } } + public void testClusterMetadataManifestXContentV2() throws IOException { + UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( + 1L, + 1L, + "test-cluster-uuid", + "test-state-uuid", + Version.CURRENT, + "test-node-id", + false, + ClusterMetadataManifest.CODEC_V2, + "test-metadata", + Collections.singletonList(uploadedIndexMetadata), + "prev-cluster-uuid", + true, + 1L, + Collections.singletonList(uploadedIndexMetadata) + ); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); + assertEquals(originalManifest, fromXContentManifest); + } + } + private List randomUploadedIndexMetadataList() { final int size = randomIntBetween(1, 10); final List uploadedIndexMetadataList = new ArrayList<>(size); diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index 068db554b4226..0f42508615a26 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -8,28 +8,22 @@ package org.opensearch.gateway.remote.routingtable; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.InputStreamDataInput; -import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.BytesStreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { - public void testWrite() throws IOException { + public void testIndexRoutingTableHeader() throws IOException { + IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); BytesStreamOutput out = new BytesStreamOutput(); - IndexRoutingTableHeader header = new IndexRoutingTableHeader(1, "dummyIndex", Version.V_3_0_0); header.write(out); BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); - CodecUtil.checkHeader(new InputStreamDataInput(in),IndexRoutingTableHeader.INDEX_ROUTING_HEADER_CODEC, IndexRoutingTableHeader.INITIAL_VERSION, IndexRoutingTableHeader.CURRENT_VERSION ); - assertEquals(1, in.readLong()); - assertEquals(Version.V_3_0_0.id, in.readInt()); - assertEquals("dummyIndex", in.readString()); + IndexRoutingTableHeader headerRead = IndexRoutingTableHeader.read(in); + assertEquals("dummyIndex", headerRead.getIndexName()); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java index 3957d4602201d..b10a60943798a 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java @@ -33,9 +33,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; -public class IndexRoutingTableInputStreamTests extends ReplicationTrackerTestCase { +public class IndexRoutingTableInputStreamTests extends OpenSearchTestCase { - public void testRoutingTableInputStream() throws IOException { + public void testRoutingTableInputStream(){ Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) .build(); @@ -44,14 +44,14 @@ public void testRoutingTableInputStream() throws IOException { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - logger.info("IndexShardRoutingTables: {}", indexShardRoutingTables); - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables, - initialRoutingTable.version(), Version.CURRENT); + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); Map indexShardRoutingTableMap = reader.read(); - logger.info("indexShardRoutingTableMap: {}", indexShardRoutingTableMap); + assertEquals(1, indexShardRoutingTableMap.size()); + assertNotNull(indexShardRoutingTableMap.get("test")); + assertEquals(2,indexShardRoutingTableMap.get("test").shards().size()); } catch (IOException e) { throw new RuntimeException(e); } From 7b2bc79266e4442a30c2daf8730ea0c615f1ec48 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Tue, 14 May 2024 13:30:14 +0530 Subject: [PATCH 13/50] Refactor reader and add failure test Signed-off-by: Arpit Bandejiya --- .../remote/ClusterMetadataManifest.java | 25 ++++++--- .../IndexRoutingTableInputStream.java | 22 ++++---- .../IndexRoutingTableInputStreamReader.java | 35 +++++------- .../IndexRoutingTableInputStreamTests.java | 54 +++++++++++-------- .../snapshots/BlobStoreFormatTests.java | 2 +- 5 files changed, 78 insertions(+), 60 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 5b7192c624fed..90ab4869ad1fd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -161,7 +161,7 @@ private static List indicesRouting(Object[] fields) { clusterUUIDCommitted(fields), routingTableVersion(fields), indicesRouting(fields) - ) + ) ); private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; @@ -287,8 +287,22 @@ public ClusterMetadataManifest( String previousClusterUUID, boolean clusterUUIDCommitted ) { - this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, - globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, -1, new ArrayList<>()); + this( + clusterTerm, + version, + clusterUUID, + stateUUID, + opensearchVersion, + nodeId, + committed, + codecVersion, + globalMetadataFileName, + indices, + previousClusterUUID, + clusterUUIDCommitted, + -1, + new ArrayList<>() + ); } public ClusterMetadataManifest( @@ -306,7 +320,7 @@ public ClusterMetadataManifest( boolean clusterUUIDCommitted, long routingTableVersion, List indicesRouting - ) { + ) { this.clusterTerm = clusterTerm; this.stateVersion = version; this.clusterUUID = clusterUUID; @@ -694,8 +708,7 @@ public String getIndexUUID() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder - .field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) + return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) .field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID()) .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index 5332a8a87c9a4..d4e2594bf153c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -61,8 +61,7 @@ public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable) throws this(indexRoutingTable, BUFFER_SIZE); } - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) - throws IOException { + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) throws IOException { this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); @@ -85,7 +84,7 @@ private void initialFill(int shardCount) throws IOException { indexRoutingTableHeader.write(out); out.writeVInt(shardCount); - System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0 , buf, 0, bytesStreamOutput.bytes().length()); + System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0, buf, 0, bytesStreamOutput.bytes().length()); count = bytesStreamOutput.bytes().length(); bytesStreamOutput.reset(); fill(buf); @@ -93,16 +92,17 @@ private void initialFill(int shardCount) throws IOException { private void fill(byte[] buf) throws IOException { if (leftOverBuf != null) { - if(leftOverBuf.length > buf.length - count) { - // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in leftOverBuf. + if (leftOverBuf.length > buf.length - count) { + // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in + // leftOverBuf. System.arraycopy(leftOverBuf, 0, buf, count, buf.length - count); - byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; - System.arraycopy(leftOverBuf, buf.length - count , tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); + byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; + System.arraycopy(leftOverBuf, buf.length - count, tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); leftOverBuf = tempLeftOverBuffer; count = buf.length - count; } else { System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); - count += leftOverBuf.length; + count += leftOverBuf.length; leftOverBuf = null; } } @@ -110,8 +110,8 @@ private void fill(byte[] buf) throws IOException { if (count < buf.length && shardIter.hasNext()) { IndexShardRoutingTable next = shardIter.next(); IndexShardRoutingTable.Builder.writeTo(next, out); - //Add checksum for the file after all shards are done - if(!shardIter.hasNext()) { + // Add checksum for the file after all shards are done + if (!shardIter.hasNext()) { out.writeLong(out.getChecksum()); } out.flush(); @@ -125,7 +125,7 @@ private void fill(byte[] buf) throws IOException { } else { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); leftOverBuf = new byte[bytesRef.length() - (buf.length - count)]; - System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count , leftOverBuf, 0, bytesRef.length() - (buf.length - count)); + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count, leftOverBuf, 0, bytesRef.length() - (buf.length - count)); count = buf.length; } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java index 35ae9f287d7f2..e2b4f5da5f6e9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -13,19 +13,13 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; -import java.io.BufferedReader; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; public class IndexRoutingTableInputStreamReader { @@ -34,32 +28,31 @@ public class IndexRoutingTableInputStreamReader { private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStreamReader.class); public IndexRoutingTableInputStreamReader(InputStream inputStream) throws IOException { - this.streamInput = new InputStreamStreamInput(inputStream); + streamInput = new InputStreamStreamInput(inputStream); } - public Map read() throws IOException { + public IndexRoutingTable readIndexRoutingTable(Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(streamInput, "assertion")) { - // Read the Table Header first - IndexRoutingTableHeader.read(in); - int shards = in.readVInt(); - logger.info("Number of Index Routing Table {}", shards); - Map indicesRouting = new HashMap(Collections.EMPTY_MAP); - for(int i=0; i indexShardRoutingTableMap = reader.read(); + IndexRoutingTable indexRoutingTable = reader.readIndexRoutingTable(metadata.index("test").getIndex()); - assertEquals(1, indexShardRoutingTableMap.size()); - assertNotNull(indexShardRoutingTableMap.get("test")); - assertEquals(2,indexShardRoutingTableMap.get("test").shards().size()); + assertEquals(1, indexRoutingTable.getShards().size()); + assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); + assertEquals(indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), 2); } catch (IOException e) { throw new RuntimeException(e); } }); } + public void testRoutingTableInputStreamWithInvalidIndex() { + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetadata.builder("invalid-index").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); + AtomicInteger assertionError = new AtomicInteger(); + initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { + try { + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); + + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + reader.readIndexRoutingTable(metadata.index("invalid-index").getIndex()); + + } catch (AssertionError e) { + assertionError.getAndIncrement(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + assertEquals(1, assertionError.get()); + } + } diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 8c64f9a3170b2..8d9a444c6f9e2 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -46,6 +46,7 @@ import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; @@ -55,7 +56,6 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.test.OpenSearchTestCase; From 88d266fb971d864943995b9a3e19270507ad4c11 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Tue, 14 May 2024 14:20:57 +0530 Subject: [PATCH 14/50] Fix GatewayMetaStatePersistedStateTests Signed-off-by: Arpit Bandejiya --- .../org/opensearch/gateway/remote/ClusterMetadataManifest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 90ab4869ad1fd..ec2e9d4169ca4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -599,6 +599,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { public Builder() { indices = new ArrayList<>(); + indicesRouting = new ArrayList<>(); } public Builder(ClusterMetadataManifest manifest) { From 4c9786925f539e543d7c8177c4c37a7f965a0a08 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 14 May 2024 16:15:09 +0530 Subject: [PATCH 15/50] Moving codec to version 2 for compatibility with manifest parser Signed-off-by: Himshikha Gupta --- .../gateway/remote/ClusterMetadataManifest.java | 4 ++++ .../gateway/remote/RemoteClusterStateService.java | 12 ++++++++++-- .../gateway/remote/ClusterMetadataManifestTests.java | 4 ++-- .../remote/RemoteClusterStateServiceTests.java | 8 +++++--- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index ec2e9d4169ca4..03512dfbf527f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -493,6 +493,10 @@ public static ClusterMetadataManifest fromXContentV0(XContentParser parser) thro return PARSER_V0.parse(parser, null); } + public static ClusterMetadataManifest fromXContentV1(XContentParser parser) throws IOException { + return PARSER_V1.parse(parser, null); + } + public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException { return CURRENT_PARSER.parse(parser, null); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index eaf607564185c..619ff7d6692ba 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -129,7 +129,13 @@ public class RemoteClusterStateService implements Closeable { new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); /** - * Manifest format compatible with codec v1, where we introduced codec versions/global metadata. + * Manifest format compatible with older codec v1, where codec versions/global metadata was introduced. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); + + /** + * Manifest format compatible with codec v2, where we introduced routing table metadata in manifest. */ public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( "cluster-metadata-manifest", @@ -172,7 +178,7 @@ public class RemoteClusterStateService implements Closeable { private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private final RemotePersistenceStats remoteStateStats; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; // ToXContent Params with gateway mode. @@ -1175,6 +1181,8 @@ private ChecksumBlobStoreFormat getClusterMetadataManif long codecVersion = getManifestCodecVersion(fileName); if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V1; } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { return CLUSTER_METADATA_MANIFEST_FORMAT_V0; } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 73933141ddf23..f9ba6415bf59f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -53,7 +53,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { } } - public void testClusterMetadataManifestXContent() throws IOException { + public void testClusterMetadataManifestXContentV1() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( 1L, @@ -75,7 +75,7 @@ public void testClusterMetadataManifestXContent() throws IOException { builder.endObject(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV1(parser); assertEquals(originalManifest, fromXContentManifest); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 9f321cd62847c..fd8d0c8bfc4a8 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -446,7 +446,7 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { * In final manifest codec version should be 1 and * global metadata should be updated, even if it was not changed in this cluster state update */ - public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOException { + public void testMigrationFromCodecV0ManifestToCodecV2Manifest() throws IOException { mockBlobStoreObjects(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -481,7 +481,7 @@ public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOExcepti // global metadata is updated assertThat(manifestAfterUpdate.getGlobalMetadataFileName(), notNullValue()); // Manifest file with codec version with 1 is updated. - assertThat(manifestAfterUpdate.getCodecVersion(), is(ClusterMetadataManifest.CODEC_V1)); + assertThat(manifestAfterUpdate.getCodecVersion(), is(ClusterMetadataManifest.CODEC_V2)); } public void testWriteIncrementalGlobalMetadataSuccess() throws IOException { @@ -803,6 +803,8 @@ public void testReadGlobalMetadata() throws IOException { .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of()) .build(); Metadata expactedMetadata = Metadata.builder().persistentSettings(Settings.builder().put("readonly", true).build()).build(); @@ -1464,7 +1466,7 @@ private void mockBlobContainerForGlobalMetadata( ClusterMetadataManifest clusterMetadataManifest, Metadata metadata ) throws IOException { - String mockManifestFileName = "manifest__1__2__C__456__1"; + String mockManifestFileName = "manifest__1__2__C__456__2"; BlobMetadata blobMetadata = new PlainBlobMetadata(mockManifestFileName, 1); when( blobContainer.listBlobsByPrefixInSortedOrder( From 2107372af0bf6e56c5a6fcc1f5964ffa2f76512c Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 15 May 2024 11:59:42 +0530 Subject: [PATCH 16/50] spotless apply and fixing gradle failures Signed-off-by: Himshikha Gupta --- .../remote/RemoteRoutingTableService.java | 26 +++++++++++++------ .../opensearch/common/util/FeatureFlags.java | 1 - .../remote/RemoteClusterStateService.java | 11 ++++---- .../remotestore/RemoteStoreNodeAttribute.java | 5 ++-- .../RemoteRoutingTableServiceTests.java | 15 +++++------ .../RemoteClusterStateServiceTests.java | 10 ++++--- 6 files changed, 39 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 7755fb5b74e4e..5392a07b8e8b3 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -52,23 +52,29 @@ public class RemoteRoutingTableService implements Closeable { private final ClusterSettings clusterSettings; private BlobStoreRepository blobStoreRepository; - public RemoteRoutingTableService(Supplier repositoriesService, - Settings settings, - ClusterSettings clusterSettings) { + public RemoteRoutingTableService( + Supplier repositoriesService, + Settings settings, + ClusterSettings clusterSettings + ) { assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; this.clusterSettings = clusterSettings; } - public List writeFullRoutingTable(ClusterState clusterState, String previousClusterUUID) { + public List writeFullRoutingTable( + ClusterState clusterState, + String previousClusterUUID + ) { return null; } public List writeIncrementalMetadata( ClusterState previousClusterState, ClusterState clusterState, - ClusterMetadataManifest previousManifest) { + ClusterMetadataManifest previousManifest + ) { return null; } @@ -76,12 +82,16 @@ public RoutingTable getLatestRoutingTable(String clusterName, String clusterUUID return null; } - public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest previousManifest, String clusterName, String clusterUUID) { + public RoutingTable getIncrementalRoutingTable( + ClusterState previousClusterState, + ClusterMetadataManifest previousManifest, + String clusterName, + String clusterUUID + ) { return null; } - private void deleteStaleRoutingTable(String clusterName, String clusterUUID, int manifestsToRetain) { - } + private void deleteStaleRoutingTable(String clusterName, String clusterUUID, int manifestsToRetain) {} @Override public void close() throws IOException { diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 708b1a7ee39ed..c98cd24af3eeb 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -104,7 +104,6 @@ public class FeatureFlags { Property.NodeScope ); - private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 3a6c428a27a04..0d2e03788dbcd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -214,9 +214,8 @@ public RemoteClusterStateService( this.remoteStateStats = new RemotePersistenceStats(); this.indexMetadataUploadListeners = indexMetadataUploadListeners; - if(isRemoteRoutingTableEnabled(settings)) { - this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, - settings, clusterSettings); + if (isRemoteRoutingTableEnabled(settings)) { + this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, settings, clusterSettings); } } @@ -652,7 +651,7 @@ public void close() throws IOException { if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } - if(this.remoteRoutingTableService != null) { + if (this.remoteRoutingTableService != null) { this.remoteRoutingTableService.close(); } } @@ -666,7 +665,7 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; - if(this.remoteRoutingTableService != null) { + if (this.remoteRoutingTableService != null) { this.remoteRoutingTableService.start(); } } @@ -805,7 +804,7 @@ public TimeValue getMetadataManifestUploadTimeout() { return this.metadataManifestUploadTimeout; } - //Package private for unit test + // Package private for unit test RemoteRoutingTableService getRemoteRoutingTableService() { return this.remoteRoutingTableService; } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 091b1903ebd4d..0538f4570006d 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -163,7 +163,7 @@ private Set getValidatedRepositoryNames(DiscoveryNode node) { } else if (node.getAttributes().containsKey(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); } - if (node.getAttributes().containsKey(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)){ + if (node.getAttributes().containsKey(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)); } @@ -203,7 +203,8 @@ public static boolean isRemoteRoutingTableAttributePresent(Settings settings) { } public static boolean isRemoteRoutingTableEnabled(Settings settings) { - return FeatureFlags.isEnabled(REMOTE_ROUTING_TABLE_EXPERIMENTAL) && RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.get(settings) + return FeatureFlags.isEnabled(REMOTE_ROUTING_TABLE_EXPERIMENTAL) + && RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.get(settings) && isRemoteRoutingTableAttributePresent(settings); } diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 4026a36e76600..8f02a61ab4ef4 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -8,8 +8,6 @@ package org.opensearch.cluster.routing.remote; -import org.junit.After; -import org.junit.Before; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -18,12 +16,16 @@ import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; import java.util.function.Supplier; -import static org.mockito.Mockito.*; import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { @@ -52,11 +54,7 @@ public void setup() { Settings nodeSettings = Settings.builder().put(REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); FeatureFlags.initializeFeatureFlags(nodeSettings); - remoteRoutingTableService = new RemoteRoutingTableService( - repositoriesServiceSupplier, - settings, - clusterSettings - ); + remoteRoutingTableService = new RemoteRoutingTableService(repositoriesServiceSupplier, settings, clusterSettings); } @After @@ -65,7 +63,6 @@ public void teardown() throws Exception { remoteRoutingTableService.close(); } - public void testFailInitializationWhenRemoteRoutingDisabled() { final Settings settings = Settings.builder().build(); assertThrows( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 63a986d779141..0b1dbf0267693 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -88,6 +88,10 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -100,7 +104,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.*; public class RemoteClusterStateServiceTests extends OpenSearchTestCase { @@ -1224,7 +1227,7 @@ public void testGlobalMetadataUploadWaitTimeSetting() { } public void testRemoteRoutingTableNotInitializedWhenDisabled() { - assertNull(remoteClusterStateService.getRemoteRoutingTableService()); + assertNull(remoteClusterStateService.getRemoteRoutingTableService()); } public void testRemoteRoutingTableInitializedWhenEnabled() { @@ -1245,7 +1248,8 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { newSettings, clusterSettings, () -> 0L, - threadPool + threadPool, + List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)) ); assertNotNull(remoteClusterStateService.getRemoteRoutingTableService()); } From 89c33bc27cc2e0911f741dec71ff5374c8b5f063 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 16 May 2024 15:52:33 +0530 Subject: [PATCH 17/50] Remove unused methods Signed-off-by: Himshikha Gupta --- .../remote/RemoteRoutingTableService.java | 34 ------------------- .../remote/RemoteClusterStateService.java | 2 +- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 5392a07b8e8b3..4bf3727682ecc 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,13 +10,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -25,7 +22,6 @@ import java.io.Closeable; import java.io.IOException; -import java.util.List; import java.util.function.Supplier; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; @@ -63,36 +59,6 @@ public RemoteRoutingTableService( this.clusterSettings = clusterSettings; } - public List writeFullRoutingTable( - ClusterState clusterState, - String previousClusterUUID - ) { - return null; - } - - public List writeIncrementalMetadata( - ClusterState previousClusterState, - ClusterState clusterState, - ClusterMetadataManifest previousManifest - ) { - return null; - } - - public RoutingTable getLatestRoutingTable(String clusterName, String clusterUUID) { - return null; - } - - public RoutingTable getIncrementalRoutingTable( - ClusterState previousClusterState, - ClusterMetadataManifest previousManifest, - String clusterName, - String clusterUUID - ) { - return null; - } - - private void deleteStaleRoutingTable(String clusterName, String clusterUUID, int manifestsToRetain) {} - @Override public void close() throws IOException { if (blobStoreRepository != null) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 1cbad7cb41b33..81294a1f9aeba 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -17,8 +17,8 @@ import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; From df6410528d220b479e4d21feec46e86c821a2bb4 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 24 May 2024 14:24:26 +0530 Subject: [PATCH 18/50] Handle routingTable repo check in JoinTaskExecutor Signed-off-by: Himshikha Gupta --- .../coordination/JoinTaskExecutor.java | 20 ++- .../metadata/RepositoriesMetadata.java | 25 ++++ .../remote/RemoteRoutingTableService.java | 5 +- .../remote/RemoteClusterStateService.java | 2 +- .../remotestore/RemoteStoreNodeAttribute.java | 18 ++- .../coordination/JoinTaskExecutorTests.java | 120 ++++++++++++++++++ .../RemoteRoutingTableServiceTests.java | 5 +- .../node/RemoteStoreNodeAttributeTests.java | 67 ++++++++++ 8 files changed, 249 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 5475470b81b93..3bea8b1655a99 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -511,11 +511,23 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod assert existingNodes.isEmpty() == false; CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); + List reposToSkip = new ArrayList<>(); + // Skip checking for remote routing table repo if feature is not enabled. + if(!RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(metadata.settings())) { + String joiningNodeRepoName = joiningNode.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); + String existingNodeRepoName = existingNodes.get(0).getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); + if(joiningNodeRepoName != null){ + reposToSkip.add(joiningNodeRepoName); + } + if(existingNodeRepoName != null) { + reposToSkip.add(existingNodeRepoName); + } + } if (STRICT.equals(remoteStoreCompatibilityMode)) { DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { - ensureRemoteStoreNodesCompatibility(joiningNode, existingNode); + ensureRemoteStoreNodesCompatibility(joiningNode, existingNode, reposToSkip); } else { if (existingNode.isRemoteStoreNode()) { throw new IllegalStateException( @@ -538,18 +550,18 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod } if (joiningNode.isRemoteStoreNode()) { Optional remoteDN = existingNodes.stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst(); - remoteDN.ifPresent(discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode)); + remoteDN.ifPresent(discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode, reposToSkip)); } } } } - private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNode existingNode) { + private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNode existingNode, List reposToSkip) { if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode); - if (existingRemoteStoreNodeAttribute.equals(joiningRemoteStoreNodeAttribute) == false) { + if (existingRemoteStoreNodeAttribute.equalsWithRepoSkip(joiningRemoteStoreNodeAttribute, reposToSkip) == false) { throw new IllegalStateException( "a remote store node [" + joiningNode diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index 9b52bdd1b16c5..3646caed9c501 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -53,6 +53,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.stream.Collectors; import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING; @@ -164,6 +165,30 @@ public boolean equalsIgnoreGenerations(@Nullable RepositoriesMetadata other) { return true; } + /** + * Checks if this instance and the give instance share the same repositories, with option to skip checking for a list of repos. + * This will support + * @param other other repositories metadata + * @param reposToSkip list of repos to skip check for equality + * @return {@code true} iff both instances contain the same repositories apart from differences in generations, not including repos provided in reposToSkip. + */ + public boolean equalsIgnoreGenerationsWithRepoSkip(@Nullable RepositoriesMetadata other, List reposToSkip) { + if (other == null) { + return false; + } + List currentRepositories = repositories.stream().filter(repo-> !reposToSkip.contains(repo.name())).collect(Collectors.toList()); + List otherRepositories = other.repositories.stream().filter(repo-> !reposToSkip.contains(repo.name())).collect(Collectors.toList()); + if (otherRepositories.size() != currentRepositories.size()) { + return false; + } + for (int i = 0; i < currentRepositories.size(); i++) { + if (currentRepositories.get(i).equalsIgnoreGenerations(otherRepositories.get(i)) == false) { + return false; + } + } + return true; + } + @Override public int hashCode() { return repositories.hashCode(); diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 4bf3727682ecc..b0c220b45c9e4 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -45,18 +45,15 @@ public class RemoteRoutingTableService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteRoutingTableService.class); private final Settings settings; private final Supplier repositoriesService; - private final ClusterSettings clusterSettings; private BlobStoreRepository blobStoreRepository; public RemoteRoutingTableService( Supplier repositoriesService, - Settings settings, - ClusterSettings clusterSettings + Settings settings ) { assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; - this.clusterSettings = clusterSettings; } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 81294a1f9aeba..86e608b5eb787 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -258,7 +258,7 @@ public RemoteClusterStateService( this.indexMetadataUploadListeners = indexMetadataUploadListeners; if (isRemoteRoutingTableEnabled(settings)) { - this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, settings, clusterSettings); + this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, settings); } } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 0538f4570006d..648f820835554 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -138,6 +138,7 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na // Repository metadata built here will always be for a system repository. settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); + settings.put("repositories.fs.location", "./fsdata"); return new RepositoryMetadata(name, type, settings.build(), cryptoMetadata); } @@ -197,7 +198,7 @@ public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { && isRemoteClusterStateAttributePresent(settings); } - public static boolean isRemoteRoutingTableAttributePresent(Settings settings) { + private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) .isEmpty() == false; } @@ -252,6 +253,21 @@ public int hashCode() { return hashCode; } + /** + * Checks if 2 instances are equal, with option to skip check for a list of repos. + * + * @param o other instance + * @param reposToSkip list of repos to skip check for equality + * @return {@code true} iff both instances are equal, not including the repositories in both instances if they are part of reposToSkip. + */ + public boolean equalsWithRepoSkip(Object o, List reposToSkip) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RemoteStoreNodeAttribute that = (RemoteStoreNodeAttribute) o; + return this.getRepositoriesMetadata().equalsIgnoreGenerationsWithRepoSkip(that.getRepositoriesMetadata(), reposToSkip); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 3e343e95f6c4b..c7a480b5060b5 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -49,10 +49,13 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -72,6 +75,7 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; @@ -944,6 +948,99 @@ public void testNodeJoinInMixedMode() { JoinTaskExecutor.ensureNodesCompatibility(joiningNode2, currentNodes, metadata); } + public void testRemoteRoutingTableDisabledNodeJoin() { + + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + + public void testRemoteRoutingTableDisabledNodeJoinRepoPresentInJoiningNode() { + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + Map attr = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + attr.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); + DiscoveryNode joiningNode = newDiscoveryNode(attr); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + + public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInExistingNode() { + Map attr = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + attr.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + attr, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + final Settings settings = Settings.builder() + .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true") + .put(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO) + .put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true") + .build(); + final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .metadata(metadata) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + assertThrows( + IllegalStateException.class, + () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) + ); + } + + public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInBothNode() { + Map attr = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + attr.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + attr, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + final Settings settings = Settings.builder() + .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true") + .put(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO) + .put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true") + .build(); + final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .metadata(metadata) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(attr); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode existingNode, int expectedRepositories) throws Exception { @@ -985,6 +1082,7 @@ private DiscoveryNode newDiscoveryNode(Map attributes) { private static final String TRANSLOG_REPO = "translog-repo"; private static final String CLUSTER_STATE_REPO = "cluster-state-repo"; private static final String COMMON_REPO = "remote-repo"; + private static final String ROUTING_TABLE_REPO = "routing-table-repo"; private Map remoteStoreNodeAttributes(String segmentRepoName, String translogRepoName) { return remoteStoreNodeAttributes(segmentRepoName, translogRepoName, CLUSTER_STATE_REPO); @@ -1049,6 +1147,28 @@ private Map remoteStateNodeAttributes(String clusterStateRepo) { }; } + private Map remoteRoutingTableAttributes(String repoName) { + String routingTableRepositoryTypeAttributeKey = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + repoName + ); + String routingTableRepositorySettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + repoName + ); + + return new HashMap<>() { + { + put(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName); + putIfAbsent(routingTableRepositoryTypeAttributeKey, "s3"); + putIfAbsent(routingTableRepositorySettingsAttributeKeyPrefix + "bucket", "state_bucket"); + putIfAbsent(routingTableRepositorySettingsAttributeKeyPrefix + "base_path", "/state/path"); + } + }; + } + private void validateAttributes(Map remoteStoreNodeAttributes, ClusterState currentState, DiscoveryNode existingNode) { DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes); Exception e = assertThrows( diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 8f02a61ab4ef4..c2f12a3e518b0 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -54,7 +54,7 @@ public void setup() { Settings nodeSettings = Settings.builder().put(REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); FeatureFlags.initializeFeatureFlags(nodeSettings); - remoteRoutingTableService = new RemoteRoutingTableService(repositoriesServiceSupplier, settings, clusterSettings); + remoteRoutingTableService = new RemoteRoutingTableService(repositoriesServiceSupplier, settings); } @After @@ -69,8 +69,7 @@ public void testFailInitializationWhenRemoteRoutingDisabled() { AssertionError.class, () -> new RemoteRoutingTableService( repositoriesServiceSupplier, - settings, - new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + settings ) ); } diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index c4ba271d27ae9..d6a1514723441 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -19,6 +19,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Arrays; import java.util.Locale; import java.util.Map; @@ -30,6 +31,7 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; public class RemoteStoreNodeAttributeTests extends OpenSearchTestCase { @@ -148,4 +150,69 @@ public void testNoCryptoMetadata() throws UnknownHostException { RepositoryMetadata repositoryMetadata = remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().get(0); assertNull(repositoryMetadata.cryptoMetadata()); } + + public void testEqualsWithRepoSkip() throws UnknownHostException { + String repoName = "remote-store-A"; + String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); + String repoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repoName); + Map attr = Map.of( + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + repoTypeSettingKey, + "s3", + repoSettingsKey, + "abc", + repoSettingsKey + "base_path", + "xyz" + ); + DiscoveryNode node = new DiscoveryNode( + "C", + new TransportAddress(InetAddress.getByName("localhost"), 9876), + attr, + emptySet(), + Version.CURRENT + ); + + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); + + String routingTableRepoName = "remote-store-B"; + String routingTableRepoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, routingTableRepoName); + String routingTableRepoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, routingTableRepoName); + + Map attr2 = Map.of( + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, + routingTableRepoName, + repoTypeSettingKey, + "s3", + repoSettingsKey, + "abc", + repoSettingsKey + "base_path", + "xyz", + routingTableRepoTypeSettingKey, + "s3", + routingTableRepoSettingsKey, + "xyz" + ); + DiscoveryNode node2 = new DiscoveryNode( + "C", + new TransportAddress(InetAddress.getByName("localhost"), 9876), + attr2, + emptySet(), + Version.CURRENT + ); + RemoteStoreNodeAttribute remoteStoreNodeAttribute2 = new RemoteStoreNodeAttribute(node2); + + assertFalse(remoteStoreNodeAttribute.equalsWithRepoSkip(remoteStoreNodeAttribute2, Arrays.asList())); + assertTrue(remoteStoreNodeAttribute.equalsWithRepoSkip(remoteStoreNodeAttribute2, Arrays.asList(routingTableRepoName))); + } } From 59a0e90f6120619fe160bce9a16a5240525ab84f Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 24 May 2024 14:42:24 +0530 Subject: [PATCH 19/50] spotless fix Signed-off-by: Himshikha Gupta --- .../coordination/JoinTaskExecutor.java | 19 +++++++++++++------ .../metadata/RepositoriesMetadata.java | 10 +++++++--- .../remote/RemoteRoutingTableService.java | 6 +----- .../coordination/JoinTaskExecutorTests.java | 10 ++++++++-- .../RemoteRoutingTableServiceTests.java | 8 +------- .../node/RemoteStoreNodeAttributeTests.java | 14 +++++++++++--- 6 files changed, 41 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 3bea8b1655a99..a111252be703a 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -513,13 +513,16 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); List reposToSkip = new ArrayList<>(); // Skip checking for remote routing table repo if feature is not enabled. - if(!RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(metadata.settings())) { - String joiningNodeRepoName = joiningNode.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); - String existingNodeRepoName = existingNodes.get(0).getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); - if(joiningNodeRepoName != null){ + if (!RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(metadata.settings())) { + String joiningNodeRepoName = joiningNode.getAttributes() + .get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); + String existingNodeRepoName = existingNodes.get(0) + .getAttributes() + .get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); + if (joiningNodeRepoName != null) { reposToSkip.add(joiningNodeRepoName); } - if(existingNodeRepoName != null) { + if (existingNodeRepoName != null) { reposToSkip.add(existingNodeRepoName); } } @@ -556,7 +559,11 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod } } - private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNode existingNode, List reposToSkip) { + private static void ensureRemoteStoreNodesCompatibility( + DiscoveryNode joiningNode, + DiscoveryNode existingNode, + List reposToSkip + ) { if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index 3646caed9c501..7356db2a1ff10 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -172,12 +172,16 @@ public boolean equalsIgnoreGenerations(@Nullable RepositoriesMetadata other) { * @param reposToSkip list of repos to skip check for equality * @return {@code true} iff both instances contain the same repositories apart from differences in generations, not including repos provided in reposToSkip. */ - public boolean equalsIgnoreGenerationsWithRepoSkip(@Nullable RepositoriesMetadata other, List reposToSkip) { + public boolean equalsIgnoreGenerationsWithRepoSkip(@Nullable RepositoriesMetadata other, List reposToSkip) { if (other == null) { return false; } - List currentRepositories = repositories.stream().filter(repo-> !reposToSkip.contains(repo.name())).collect(Collectors.toList()); - List otherRepositories = other.repositories.stream().filter(repo-> !reposToSkip.contains(repo.name())).collect(Collectors.toList()); + List currentRepositories = repositories.stream() + .filter(repo -> !reposToSkip.contains(repo.name())) + .collect(Collectors.toList()); + List otherRepositories = other.repositories.stream() + .filter(repo -> !reposToSkip.contains(repo.name())) + .collect(Collectors.toList()); if (otherRepositories.size() != currentRepositories.size()) { return false; } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index b0c220b45c9e4..5d6392be37ba6 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; @@ -47,10 +46,7 @@ public class RemoteRoutingTableService implements Closeable { private final Supplier repositoriesService; private BlobStoreRepository blobStoreRepository; - public RemoteRoutingTableService( - Supplier repositoriesService, - Settings settings - ) { + public RemoteRoutingTableService(Supplier repositoriesService, Settings settings) { assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index c7a480b5060b5..6dd5508bca589 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -996,7 +996,10 @@ public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInExistingNode() { ); final Settings settings = Settings.builder() .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true") - .put(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO) + .put( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, + ROUTING_TABLE_REPO + ) .put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true") .build(); final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); @@ -1026,7 +1029,10 @@ public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInBothNode() { ); final Settings settings = Settings.builder() .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true") - .put(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO) + .put( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, + ROUTING_TABLE_REPO + ) .put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true") .build(); final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index c2f12a3e518b0..040fde227dc4e 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -65,13 +65,7 @@ public void teardown() throws Exception { public void testFailInitializationWhenRemoteRoutingDisabled() { final Settings settings = Settings.builder().build(); - assertThrows( - AssertionError.class, - () -> new RemoteRoutingTableService( - repositoriesServiceSupplier, - settings - ) - ); + assertThrows(AssertionError.class, () -> new RemoteRoutingTableService(repositoriesServiceSupplier, settings)); } public void testFailStartWhenRepositoryNotSet() { diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index d6a1514723441..1ee5b5540f07c 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -29,9 +29,9 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; public class RemoteStoreNodeAttributeTests extends OpenSearchTestCase { @@ -180,8 +180,16 @@ public void testEqualsWithRepoSkip() throws UnknownHostException { RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); String routingTableRepoName = "remote-store-B"; - String routingTableRepoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, routingTableRepoName); - String routingTableRepoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, routingTableRepoName); + String routingTableRepoTypeSettingKey = String.format( + Locale.ROOT, + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + routingTableRepoName + ); + String routingTableRepoSettingsKey = String.format( + Locale.ROOT, + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + routingTableRepoName + ); Map attr2 = Map.of( REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, From 51333c9a2e2e466e8eead84d49122f9a16efe9ab Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 24 May 2024 15:55:00 +0530 Subject: [PATCH 20/50] Reverting accidental commit Signed-off-by: Himshikha Gupta --- .../opensearch/node/remotestore/RemoteStoreNodeAttribute.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 648f820835554..d579d6ed82ede 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -138,7 +138,6 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na // Repository metadata built here will always be for a system repository. settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); - settings.put("repositories.fs.location", "./fsdata"); return new RepositoryMetadata(name, type, settings.build(), cryptoMetadata); } From 418d9fd5787806449b51733612c95389e056e25e Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 28 May 2024 16:07:50 +0530 Subject: [PATCH 21/50] Removing buffer logic Signed-off-by: Himshikha Gupta --- .../routingtable/IndexRoutingTableInput.java | 46 +++++ .../IndexRoutingTableInputStream.java | 173 ------------------ ....java => IndexRoutingTableInputTests.java} | 13 +- 3 files changed, 52 insertions(+), 180 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java rename server/src/test/java/org/opensearch/gateway/remote/routingtable/{IndexRoutingTableInputStreamTests.java => IndexRoutingTableInputTests.java} (85%) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java new file mode 100644 index 0000000000000..937de104fa5ad --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; + +import java.io.IOException; +import java.util.Iterator; + +public class IndexRoutingTableInput { + + private final IndexRoutingTableHeader indexRoutingTableHeader; + private final Iterator shardIter; + private int shardCount; + + public IndexRoutingTableInput(IndexRoutingTable indexRoutingTable) { + this.shardIter = indexRoutingTable.iterator(); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); + this.shardCount = indexRoutingTable.shards().size(); + } + + public BytesReference write() throws IOException { + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput); + indexRoutingTableHeader.write(out); + out.writeVInt(shardCount); + while (shardIter.hasNext()) { + IndexShardRoutingTable next = shardIter.next(); + IndexShardRoutingTable.Builder.writeTo(next, out); + } + out.writeLong(out.getChecksum()); + out.flush(); + return bytesStreamOutput.bytes(); + } + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java deleted file mode 100644 index d4e2594bf153c..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.routingtable; - -import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Iterator; - -public class IndexRoutingTableInputStream extends InputStream { - - /** - * The buffer where data is stored. - */ - protected byte[] buf; - - /** - * The number of valid bytes in the buffer. - */ - protected int count; - - /** - * The buffer left over from the last fill - */ - protected byte[] leftOverBuf; - - /** - * The mark position - */ - protected int markPos = -1; - - /** - * The read limit - */ - protected int markLimit; - - /** - * The position - */ - protected int pos; - - private static final int BUFFER_SIZE = 8192; - - private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; - private final BytesStreamOutput bytesStreamOutput; - private final BufferedChecksumStreamOutput out; - - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable) throws IOException { - this(indexRoutingTable, BUFFER_SIZE); - } - - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) throws IOException { - this.buf = new byte[size]; - this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); - this.bytesStreamOutput = new BytesStreamOutput(); - this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); - - initialFill(indexRoutingTable.shards().size()); - } - - @Override - public int read() throws IOException { - if (pos >= count) { - maybeResizeAndFill(); - if (pos >= count) return -1; - } - return buf[pos++] & 0xff; - } - - private void initialFill(int shardCount) throws IOException { - indexRoutingTableHeader.write(out); - out.writeVInt(shardCount); - - System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0, buf, 0, bytesStreamOutput.bytes().length()); - count = bytesStreamOutput.bytes().length(); - bytesStreamOutput.reset(); - fill(buf); - } - - private void fill(byte[] buf) throws IOException { - if (leftOverBuf != null) { - if (leftOverBuf.length > buf.length - count) { - // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in - // leftOverBuf. - System.arraycopy(leftOverBuf, 0, buf, count, buf.length - count); - byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; - System.arraycopy(leftOverBuf, buf.length - count, tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); - leftOverBuf = tempLeftOverBuffer; - count = buf.length - count; - } else { - System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); - count += leftOverBuf.length; - leftOverBuf = null; - } - } - - if (count < buf.length && shardIter.hasNext()) { - IndexShardRoutingTable next = shardIter.next(); - IndexShardRoutingTable.Builder.writeTo(next, out); - // Add checksum for the file after all shards are done - if (!shardIter.hasNext()) { - out.writeLong(out.getChecksum()); - } - out.flush(); - BytesReference bytesRef = bytesStreamOutput.bytes(); - bytesStreamOutput.reset(); - - if (bytesRef.length() < buf.length - count) { - System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, bytesRef.length()); - count += bytesRef.length(); - leftOverBuf = null; - } else { - System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); - leftOverBuf = new byte[bytesRef.length() - (buf.length - count)]; - System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count, leftOverBuf, 0, bytesRef.length() - (buf.length - count)); - count = buf.length; - } - } - } - - private void maybeResizeAndFill() throws IOException { - byte[] buffer = buf; - if (markPos == -1) pos = 0; /* no mark: throw away the buffer */ - else if (pos >= buffer.length) { /* no room left in buffer */ - if (markPos > 0) { /* can throw away early part of the buffer */ - int sz = pos - markPos; - System.arraycopy(buffer, markPos, buffer, 0, sz); - pos = sz; - markPos = 0; - } else if (buffer.length >= markLimit) { - markPos = -1; /* buffer got too big, invalidate mark */ - pos = 0; /* drop buffer contents */ - } else { /* grow buffer */ - int nsz = markLimit + 1; - byte[] nbuf = new byte[nsz]; - System.arraycopy(buffer, 0, nbuf, 0, pos); - buffer = nbuf; - } - } - count = pos; - fill(buffer); - } - - @Override - public void mark(int readlimit) { - markLimit = readlimit; - markPos = pos; - } - - @Override - public boolean markSupported() { - return true; - } - - @Override - public void reset() throws IOException { - if (markPos < 0) throw new IOException("Resetting to invalid mark"); - pos = markPos; - } -} diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java similarity index 85% rename from server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java rename to server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java index a6e2e4bda00e8..a37883ed7a5a5 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java @@ -17,12 +17,11 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.atomic.AtomicInteger; -public class IndexRoutingTableInputStreamTests extends OpenSearchTestCase { +public class IndexRoutingTableInputTests extends OpenSearchTestCase { - public void testRoutingTableInputStream() { + public void testRoutingTableInput() { Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) .build(); @@ -31,9 +30,9 @@ public void testRoutingTableInputStream() { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); + IndexRoutingTableInput indexRouting = new IndexRoutingTableInput(indexShardRoutingTables); - IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRouting.write().streamInput()); IndexRoutingTable indexRoutingTable = reader.readIndexRoutingTable(metadata.index("test").getIndex()); assertEquals(1, indexRoutingTable.getShards().size()); @@ -55,9 +54,9 @@ public void testRoutingTableInputStreamWithInvalidIndex() { AtomicInteger assertionError = new AtomicInteger(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); + IndexRoutingTableInput indexRouting = new IndexRoutingTableInput(indexShardRoutingTables); - IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRouting.write().streamInput()); reader.readIndexRoutingTable(metadata.index("invalid-index").getIndex()); } catch (AssertionError e) { From 802e04f7b37e11f91c0214869adbb6faf177b0de Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 28 May 2024 23:21:04 +0530 Subject: [PATCH 22/50] Move repo check logic to RemoteStoreNodeAttr Signed-off-by: Himshikha Gupta --- .../coordination/JoinTaskExecutor.java | 39 +++++---------- .../metadata/RepositoriesMetadata.java | 2 +- .../common/settings/FeatureFlagSettings.java | 2 +- .../opensearch/common/util/FeatureFlags.java | 8 ++-- .../remote/RemoteClusterStateService.java | 19 ++++---- .../main/java/org/opensearch/node/Node.java | 2 +- .../remotestore/RemoteStoreNodeAttribute.java | 29 ++++++++---- .../remotestore/RemoteStoreNodeService.java | 14 ++++-- .../coordination/JoinTaskExecutorTests.java | 8 ++-- .../RemoteRoutingTableServiceTests.java | 4 +- .../RemoteClusterStateServiceTests.java | 8 ++-- .../node/RemoteStoreNodeAttributeTests.java | 47 ++++++++++++++----- 12 files changed, 106 insertions(+), 76 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index a111252be703a..cb428eb9c3e45 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -187,7 +187,8 @@ public ClusterTasksResult execute(ClusterState currentState, List jo DiscoveryNode dn = remoteDN.orElseGet(() -> (currentNodes.getNodes().values()).stream().findFirst().get()); RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( dn, - currentState.getMetadata().custom(RepositoriesMetadata.TYPE) + currentState.getMetadata().custom(RepositoriesMetadata.TYPE), + currentState.getMetadata().settings() ); assert nodesBuilder.isLocalNodeElectedClusterManager(); @@ -223,7 +224,8 @@ public ClusterTasksResult execute(ClusterState currentState, List jo logger.info("Updating system repository now for remote store"); repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( node, - currentState.getMetadata().custom(RepositoriesMetadata.TYPE) + currentState.getMetadata().custom(RepositoriesMetadata.TYPE), + currentState.getMetadata().settings() ); } @@ -511,26 +513,11 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod assert existingNodes.isEmpty() == false; CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); - List reposToSkip = new ArrayList<>(); - // Skip checking for remote routing table repo if feature is not enabled. - if (!RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(metadata.settings())) { - String joiningNodeRepoName = joiningNode.getAttributes() - .get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); - String existingNodeRepoName = existingNodes.get(0) - .getAttributes() - .get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); - if (joiningNodeRepoName != null) { - reposToSkip.add(joiningNodeRepoName); - } - if (existingNodeRepoName != null) { - reposToSkip.add(existingNodeRepoName); - } - } if (STRICT.equals(remoteStoreCompatibilityMode)) { DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { - ensureRemoteStoreNodesCompatibility(joiningNode, existingNode, reposToSkip); + ensureRemoteStoreNodesCompatibility(joiningNode, existingNode, metadata.settings()); } else { if (existingNode.isRemoteStoreNode()) { throw new IllegalStateException( @@ -553,22 +540,20 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod } if (joiningNode.isRemoteStoreNode()) { Optional remoteDN = existingNodes.stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst(); - remoteDN.ifPresent(discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode, reposToSkip)); + remoteDN.ifPresent( + discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode, metadata.settings()) + ); } } } } - private static void ensureRemoteStoreNodesCompatibility( - DiscoveryNode joiningNode, - DiscoveryNode existingNode, - List reposToSkip - ) { + private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNode existingNode, Settings settings) { if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { - RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); - RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode); - if (existingRemoteStoreNodeAttribute.equalsWithRepoSkip(joiningRemoteStoreNodeAttribute, reposToSkip) == false) { + RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode, settings); + RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode, settings); + if (existingRemoteStoreNodeAttribute.equalsIgnoreOptionalRepo(joiningRemoteStoreNodeAttribute) == false) { throw new IllegalStateException( "a remote store node [" + joiningNode diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index 7356db2a1ff10..2c9eac78112d2 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -172,7 +172,7 @@ public boolean equalsIgnoreGenerations(@Nullable RepositoriesMetadata other) { * @param reposToSkip list of repos to skip check for equality * @return {@code true} iff both instances contain the same repositories apart from differences in generations, not including repos provided in reposToSkip. */ - public boolean equalsIgnoreGenerationsWithRepoSkip(@Nullable RepositoriesMetadata other, List reposToSkip) { + public boolean equalsIgnoreGenerationsIgnoreOptionalRepos(@Nullable RepositoriesMetadata other, List reposToSkip) { if (other == null) { return false; } diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index ee995c4649d08..238df1bd90113 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -37,6 +37,6 @@ protected FeatureFlagSettings( FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, FeatureFlags.PLUGGABLE_CACHE_SETTING, - FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING + FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index eeafe736f7dd6..82f43921d2d28 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -70,7 +70,7 @@ public class FeatureFlags { /** * Gates the functionality of remote routing table. */ - public static final String REMOTE_ROUTING_TABLE_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.routing_table.enabled"; + public static final String REMOTE_PUBLICATION_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.publication.enabled"; public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_STORE_MIGRATION_EXPERIMENTAL, @@ -94,8 +94,8 @@ public class FeatureFlags { public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); - public static final Setting REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING = Setting.boolSetting( - REMOTE_ROUTING_TABLE_EXPERIMENTAL, + public static final Setting REMOTE_PUBLICATION_EXPERIMENTAL_SETTING = Setting.boolSetting( + REMOTE_PUBLICATION_EXPERIMENTAL, false, Property.NodeScope ); @@ -108,7 +108,7 @@ public class FeatureFlags { DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, PLUGGABLE_CACHE_SETTING, - REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING + REMOTE_PUBLICATION_EXPERIMENTAL_SETTING ); /** * Should store the settings from opensearch.yml. diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 86e608b5eb787..03064de793c9d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -204,7 +204,7 @@ public class RemoteClusterStateService implements Closeable { private final List indexMetadataUploadListeners; private BlobStoreRepository blobStoreRepository; private BlobStoreTransferService blobStoreTransferService; - private RemoteRoutingTableService remoteRoutingTableService; + private Optional remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; private volatile TimeValue indexMetadataUploadTimeout; @@ -256,10 +256,9 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); this.indexMetadataUploadListeners = indexMetadataUploadListeners; - - if (isRemoteRoutingTableEnabled(settings)) { - this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, settings); - } + this.remoteRoutingTableService = isRemoteRoutingTableEnabled(settings) + ? Optional.of(new RemoteRoutingTableService(repositoriesService, settings)) + : Optional.empty(); } private BlobStoreTransferService getBlobStoreTransferService() { @@ -756,8 +755,8 @@ public void close() throws IOException { if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } - if (this.remoteRoutingTableService != null) { - this.remoteRoutingTableService.close(); + if (this.remoteRoutingTableService.isPresent()) { + this.remoteRoutingTableService.get().close(); } } @@ -770,8 +769,8 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; - if (this.remoteRoutingTableService != null) { - this.remoteRoutingTableService.start(); + if (this.remoteRoutingTableService.isPresent()) { + this.remoteRoutingTableService.get().start(); } } @@ -947,7 +946,7 @@ public TimeValue getMetadataManifestUploadTimeout() { } // Package private for unit test - RemoteRoutingTableService getRemoteRoutingTableService() { + Optional getRemoteRoutingTableService() { return this.remoteRoutingTableService; } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 9462aeddbd0e4..cad1772fe7c4d 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1976,7 +1976,7 @@ public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { ); if (isRemoteStoreAttributePresent(settings)) { - remoteStoreNodeService.createAndVerifyRepositories(discoveryNode); + remoteStoreNodeService.createAndVerifyRepositories(discoveryNode, settings); } localNode.set(discoveryNode); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index d579d6ed82ede..e35229f02a554 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -30,7 +30,7 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; +import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; /** * This is an abstraction for validating and storing information specific to remote backed storage nodes. @@ -53,6 +53,7 @@ public class RemoteStoreNodeAttribute { public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing_table.repository"; private final RepositoriesMetadata repositoriesMetadata; + private final List optionalRepos; public static List SUPPORTED_DATA_REPO_NAME_ATTRIBUTES = List.of( REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, @@ -62,8 +63,18 @@ public class RemoteStoreNodeAttribute { /** * Creates a new {@link RemoteStoreNodeAttribute} */ - public RemoteStoreNodeAttribute(DiscoveryNode node) { + public RemoteStoreNodeAttribute(DiscoveryNode node, Settings settings) { this.repositoriesMetadata = buildRepositoriesMetadata(node); + // For supporting feature launches where new repos are added, we can mark repos to be optional and ensure node joins are not + // impacted due to diff in repos. + this.optionalRepos = new ArrayList<>(); + if (!RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings)) { + if (node.getAttributes().containsKey(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { + optionalRepos.add( + node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) + ); + } + } } private String validateAttributeNonNull(DiscoveryNode node, String attributeKey) { @@ -203,7 +214,7 @@ private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { } public static boolean isRemoteRoutingTableEnabled(Settings settings) { - return FeatureFlags.isEnabled(REMOTE_ROUTING_TABLE_EXPERIMENTAL) + return FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) && RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.get(settings) && isRemoteRoutingTableAttributePresent(settings); } @@ -253,18 +264,20 @@ public int hashCode() { } /** - * Checks if 2 instances are equal, with option to skip check for a list of repos. + * Checks if 2 instances are equal, ignoring optionalRepos for both. * * @param o other instance - * @param reposToSkip list of repos to skip check for equality - * @return {@code true} iff both instances are equal, not including the repositories in both instances if they are part of reposToSkip. + * @return {@code true} iff both instances are equal, not including the repositories in both instances if they are part of optionalRepos. */ - public boolean equalsWithRepoSkip(Object o, List reposToSkip) { + public boolean equalsIgnoreOptionalRepo(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RemoteStoreNodeAttribute that = (RemoteStoreNodeAttribute) o; - return this.getRepositoriesMetadata().equalsIgnoreGenerationsWithRepoSkip(that.getRepositoriesMetadata(), reposToSkip); + List reposToSkip = new ArrayList<>(); + reposToSkip.addAll(this.optionalRepos); + reposToSkip.addAll(that.optionalRepos); + return this.getRepositoriesMetadata().equalsIgnoreGenerationsIgnoreOptionalRepos(that.getRepositoriesMetadata(), reposToSkip); } @Override diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 874c9408de6c5..42d8fcde5cbff 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -144,8 +145,8 @@ public RemoteStoreNodeService(Supplier repositoriesService, * If the creation or verification fails this will close all the repositories this method created and throw * exception. */ - public void createAndVerifyRepositories(DiscoveryNode localNode) { - RemoteStoreNodeAttribute nodeAttribute = new RemoteStoreNodeAttribute(localNode); + public void createAndVerifyRepositories(DiscoveryNode localNode, Settings settings) { + RemoteStoreNodeAttribute nodeAttribute = new RemoteStoreNodeAttribute(localNode, settings); RepositoriesService reposService = repositoriesService.get(); Map repositories = new HashMap<>(); for (RepositoryMetadata repositoryMetadata : nodeAttribute.getRepositoriesMetadata().repositories()) { @@ -177,10 +178,15 @@ public void createAndVerifyRepositories(DiscoveryNode localNode) { * repository is already present in the cluster state and if it's different then the joining remote store backed * node repository metadata an exception will be thrown and the node will not be allowed to join the cluster. */ - public RepositoriesMetadata updateRepositoriesMetadata(DiscoveryNode joiningNode, RepositoriesMetadata existingRepositories) { + public RepositoriesMetadata updateRepositoriesMetadata( + DiscoveryNode joiningNode, + RepositoriesMetadata existingRepositories, + Settings settings + ) { if (joiningNode.isRemoteStoreNode()) { List updatedRepositoryMetadataList = new ArrayList<>(); - List newRepositoryMetadataList = new RemoteStoreNodeAttribute(joiningNode).getRepositoriesMetadata() + List newRepositoryMetadataList = new RemoteStoreNodeAttribute(joiningNode, settings) + .getRepositoriesMetadata() .repositories(); if (existingRepositories == null) { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 6dd5508bca589..6980e9e670a19 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -205,7 +205,9 @@ public void testUpdatesNodeWithNewRoles() throws Exception { when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); final RemoteStoreNodeService remoteStoreNodeService = mock(RemoteStoreNodeService.class); - when(remoteStoreNodeService.updateRepositoriesMetadata(any(), any())).thenReturn(new RepositoriesMetadata(Collections.emptyList())); + when(remoteStoreNodeService.updateRepositoriesMetadata(any(), any(), any())).thenReturn( + new RepositoriesMetadata(Collections.emptyList()) + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -1002,7 +1004,7 @@ public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInExistingNode() { ) .put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true") .build(); - final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); FeatureFlags.initializeFeatureFlags(nodeSettings); Metadata metadata = Metadata.builder().persistentSettings(settings).build(); ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) @@ -1035,7 +1037,7 @@ public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInBothNode() { ) .put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true") .build(); - final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); FeatureFlags.initializeFeatureFlags(nodeSettings); Metadata metadata = Metadata.builder().persistentSettings(settings).build(); ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 040fde227dc4e..d80a927914025 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -21,7 +21,7 @@ import java.util.function.Supplier; -import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; +import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -51,7 +51,7 @@ public void setup() { blobStoreRepository = mock(BlobStoreRepository.class); when(repositoriesService.repository("routing_repository")).thenReturn(blobStoreRepository); - Settings nodeSettings = Settings.builder().put(REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); FeatureFlags.initializeFeatureFlags(nodeSettings); remoteRoutingTableService = new RemoteRoutingTableService(repositoriesServiceSupplier, settings); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index e6d6441fd8520..9cfb1318f0f8a 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -88,7 +88,7 @@ import org.mockito.ArgumentMatchers; import static java.util.stream.Collectors.toList; -import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; +import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; @@ -1501,7 +1501,7 @@ public void testGlobalMetadataUploadWaitTimeSetting() { } public void testRemoteRoutingTableNotInitializedWhenDisabled() { - assertNull(remoteClusterStateService.getRemoteRoutingTableService()); + assertFalse(remoteClusterStateService.getRemoteRoutingTableService().isPresent()); } public void testRemoteRoutingTableInitializedWhenEnabled() { @@ -1513,7 +1513,7 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { .build(); clusterSettings.applySettings(newSettings); - Settings nodeSettings = Settings.builder().put(REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); FeatureFlags.initializeFeatureFlags(nodeSettings); remoteClusterStateService = new RemoteClusterStateService( @@ -1525,7 +1525,7 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { threadPool, List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)) ); - assertNotNull(remoteClusterStateService.getRemoteRoutingTableService()); + assertTrue(remoteClusterStateService.getRemoteRoutingTableService().isPresent()); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index 1ee5b5540f07c..eb293287a36c3 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -12,14 +12,15 @@ import org.opensearch.cluster.metadata.CryptoMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.test.OpenSearchTestCase; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.Arrays; import java.util.Locale; import java.util.Map; @@ -74,7 +75,7 @@ public void testCryptoMetadata() throws UnknownHostException { Version.CURRENT ); - RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node, Settings.builder().build()); assertEquals(remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().size(), 1); RepositoryMetadata repositoryMetadata = remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().get(0); Settings.Builder settings = Settings.builder(); @@ -116,7 +117,7 @@ public void testInvalidCryptoMetadata() throws UnknownHostException { Version.CURRENT ); - assertThrows(IllegalStateException.class, () -> new RemoteStoreNodeAttribute(node)); + assertThrows(IllegalStateException.class, () -> new RemoteStoreNodeAttribute(node, Settings.builder().build())); } public void testNoCryptoMetadata() throws UnknownHostException { @@ -145,13 +146,14 @@ public void testNoCryptoMetadata() throws UnknownHostException { Version.CURRENT ); - RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node, Settings.builder().build()); assertEquals(remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().size(), 1); RepositoryMetadata repositoryMetadata = remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().get(0); assertNull(repositoryMetadata.cryptoMetadata()); } - public void testEqualsWithRepoSkip() throws UnknownHostException { + public void testEqualsIgnoreOptionalRepo() throws UnknownHostException { + // Node 1 -> Remote Routing disabled, repo not present, Node 2 -> Remote Routing disabled, repo present -> should succeed String repoName = "remote-store-A"; String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); String repoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repoName); @@ -169,7 +171,7 @@ public void testEqualsWithRepoSkip() throws UnknownHostException { repoSettingsKey + "base_path", "xyz" ); - DiscoveryNode node = new DiscoveryNode( + DiscoveryNode nodeWithoutRoutingTableAttr = new DiscoveryNode( "C", new TransportAddress(InetAddress.getByName("localhost"), 9876), attr, @@ -177,7 +179,10 @@ public void testEqualsWithRepoSkip() throws UnknownHostException { Version.CURRENT ); - RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute( + nodeWithoutRoutingTableAttr, + Settings.builder().build() + ); String routingTableRepoName = "remote-store-B"; String routingTableRepoTypeSettingKey = String.format( @@ -211,16 +216,36 @@ public void testEqualsWithRepoSkip() throws UnknownHostException { routingTableRepoSettingsKey, "xyz" ); - DiscoveryNode node2 = new DiscoveryNode( + DiscoveryNode nodeWithRoutingTableAttr = new DiscoveryNode( "C", new TransportAddress(InetAddress.getByName("localhost"), 9876), attr2, emptySet(), Version.CURRENT ); - RemoteStoreNodeAttribute remoteStoreNodeAttribute2 = new RemoteStoreNodeAttribute(node2); + RemoteStoreNodeAttribute remoteStoreNodeAttribute2 = new RemoteStoreNodeAttribute( + nodeWithRoutingTableAttr, + Settings.builder().build() + ); + assertTrue(remoteStoreNodeAttribute.equalsIgnoreOptionalRepo(remoteStoreNodeAttribute2)); + + // Node 1 -> Remote Routing enabled, repo present, Node 2 -> Remote Routing enabled, repo present -> should succeed + final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + RemoteStoreNodeAttribute remoteStoreNodeAttribute3 = new RemoteStoreNodeAttribute( + nodeWithRoutingTableAttr, + Settings.builder() + .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true") + .put(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName) + .build() + ); + assertTrue(remoteStoreNodeAttribute3.equalsIgnoreOptionalRepo(remoteStoreNodeAttribute2)); - assertFalse(remoteStoreNodeAttribute.equalsWithRepoSkip(remoteStoreNodeAttribute2, Arrays.asList())); - assertTrue(remoteStoreNodeAttribute.equalsWithRepoSkip(remoteStoreNodeAttribute2, Arrays.asList(routingTableRepoName))); + // Node 1 -> Remote Routing enabled, repo present, Node 2 -> Remote Routing enabled, repo not present -> should fail + RemoteStoreNodeAttribute remoteStoreNodeAttribute4 = new RemoteStoreNodeAttribute( + nodeWithoutRoutingTableAttr, + Settings.builder().put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true").build() + ); + assertFalse(remoteStoreNodeAttribute3.equalsIgnoreOptionalRepo(remoteStoreNodeAttribute4)); } } From 7f33ba27989a7c5e4a7648644fdf2aa712324b2d Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 29 May 2024 13:07:16 +0530 Subject: [PATCH 23/50] Move BufferedChecksum streams to libs/core Signed-off-by: Himshikha Gupta --- .../core}/common/io/stream/BufferedChecksumStreamInput.java | 4 +--- .../core}/common/io/stream/BufferedChecksumStreamOutput.java | 2 +- .../gateway/remote/routingtable/IndexRoutingTableInput.java | 2 +- .../routingtable/IndexRoutingTableInputStreamReader.java | 2 +- .../org/opensearch/index/translog/BaseTranslogReader.java | 2 +- .../src/main/java/org/opensearch/index/translog/Translog.java | 4 ++-- .../java/org/opensearch/index/translog/TranslogHeader.java | 4 ++-- .../java/org/opensearch/index/translog/TranslogSnapshot.java | 2 +- .../java/org/opensearch/index/translog/TranslogWriter.java | 2 +- .../java/org/opensearch/snapshots/BlobStoreFormatTests.java | 2 +- 10 files changed, 12 insertions(+), 14 deletions(-) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/BufferedChecksumStreamInput.java (96%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/BufferedChecksumStreamOutput.java (98%) diff --git a/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamInput.java similarity index 96% rename from server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamInput.java index f3341712275f9..41680961b36e9 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamInput.java @@ -30,12 +30,10 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.apache.lucene.util.BitUtil; -import org.opensearch.core.common.io.stream.FilterStreamInput; -import org.opensearch.core.common.io.stream.StreamInput; import java.io.EOFException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java similarity index 98% rename from server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 254f228f1c739..0f370f2d2f1fe 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java index 937de104fa5ad..bd143334b9f28 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java @@ -10,7 +10,7 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java index e2b4f5da5f6e9..d1c3d22a2cc34 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; diff --git a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java index 9088eb6b20fb8..37af1dcbeab8b 100644 --- a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java +++ b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java @@ -32,7 +32,7 @@ package org.opensearch.index.translog; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 18b10cb886996..01c1697d75e3b 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -38,8 +38,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index c622214a3a69d..66a9fe08d06b5 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -40,8 +40,8 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.Channels; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java index a6e322bf58fff..521472f4d64a0 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java @@ -32,7 +32,7 @@ package org.opensearch.index.translog; import org.opensearch.common.io.Channels; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.index.seqno.SequenceNumbers; import java.io.EOFException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index da3c7a8dee219..2176b9ce40687 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -42,7 +42,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Channels; import org.opensearch.common.io.DiskIoBufferPool; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 8d9a444c6f9e2..0f020a73cbf7f 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -46,7 +46,7 @@ import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.io.Streams; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; From 42efb38774ffa1f877254faf9515a922efda0567 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 29 May 2024 13:18:41 +0530 Subject: [PATCH 24/50] Spotless fix Signed-off-by: Himshikha Gupta --- .../gateway/remote/routingtable/IndexRoutingTableInput.java | 2 +- .../src/main/java/org/opensearch/index/translog/Translog.java | 4 ++-- .../java/org/opensearch/index/translog/TranslogWriter.java | 2 +- .../java/org/opensearch/snapshots/BlobStoreFormatTests.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java index bd143334b9f28..3790351e5f7fa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java @@ -10,9 +10,9 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import java.io.IOException; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 01c1697d75e3b..87e0c21b8203c 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -38,8 +38,6 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; @@ -50,6 +48,8 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.index.shard.ShardId; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index 2176b9ce40687..b0c7d51c3e43b 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -42,7 +42,6 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Channels; import org.opensearch.common.io.DiskIoBufferPool; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; @@ -51,6 +50,7 @@ import org.opensearch.core.Assertions; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 0f020a73cbf7f..95a8267734a07 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -46,10 +46,10 @@ import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.io.Streams; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.CompressorRegistry; import org.opensearch.core.xcontent.ToXContent; From 9b50f8a5226482752effe43ea402469c5ff0ddf6 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Wed, 29 May 2024 14:41:27 +0530 Subject: [PATCH 25/50] Refactor RemoteIndexRoutingTable read Signed-off-by: Arpit Bandejiya --- .../routingtable/IndexRoutingTableInput.java | 46 ------------------- ...ava => RemoteIndexRoutingTableObject.java} | 42 +++++++++++------ ...> RemoteIndexRoutingTableObjectTests.java} | 34 ++++++++------ 3 files changed, 49 insertions(+), 73 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java rename server/src/main/java/org/opensearch/gateway/remote/routingtable/{IndexRoutingTableInputStreamReader.java => RemoteIndexRoutingTableObject.java} (56%) rename server/src/test/java/org/opensearch/gateway/remote/routingtable/{IndexRoutingTableInputTests.java => RemoteIndexRoutingTableObjectTests.java} (62%) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java deleted file mode 100644 index 3790351e5f7fa..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.routingtable; - -import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; - -import java.io.IOException; -import java.util.Iterator; - -public class IndexRoutingTableInput { - - private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; - private int shardCount; - - public IndexRoutingTableInput(IndexRoutingTable indexRoutingTable) { - this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); - this.shardCount = indexRoutingTable.shards().size(); - } - - public BytesReference write() throws IOException { - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput); - indexRoutingTableHeader.write(out); - out.writeVInt(shardCount); - while (shardIter.hasNext()) { - IndexShardRoutingTable next = shardIter.next(); - IndexShardRoutingTable.Builder.writeTo(next, out); - } - out.writeLong(out.getChecksum()); - out.flush(); - return bytesStreamOutput.bytes(); - } - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java similarity index 56% rename from server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java rename to server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java index d1c3d22a2cc34..263fb29120553 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java @@ -8,44 +8,58 @@ package org.opensearch.gateway.remote.routingtable; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; -import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; +import java.util.Iterator; -public class IndexRoutingTableInputStreamReader { +public class RemoteIndexRoutingTableObject { - private final StreamInput streamInput; + private final IndexRoutingTableHeader indexRoutingTableHeader; + private final Iterator shardIter; + private final int shardCount; - private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStreamReader.class); + public RemoteIndexRoutingTableObject(IndexRoutingTable indexRoutingTable) { + this.shardIter = indexRoutingTable.iterator(); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); + this.shardCount = indexRoutingTable.shards().size(); + } - public IndexRoutingTableInputStreamReader(InputStream inputStream) throws IOException { - streamInput = new InputStreamStreamInput(inputStream); + public BytesReference write() throws IOException { + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput); + indexRoutingTableHeader.write(out); + out.writeVInt(shardCount); + while (shardIter.hasNext()) { + IndexShardRoutingTable next = shardIter.next(); + IndexShardRoutingTable.Builder.writeTo(next, out); + } + out.writeLong(out.getChecksum()); + out.flush(); + return bytesStreamOutput.bytes(); } - public IndexRoutingTable readIndexRoutingTable(Index index) throws IOException { + public static IndexRoutingTable read(InputStream inputStream, Index index) throws IOException { try { - try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(streamInput, "assertion")) { + try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) { // Read the Table Header first and confirm the index IndexRoutingTableHeader indexRoutingTableHeader = IndexRoutingTableHeader.read(in); assert indexRoutingTableHeader.getIndexName().equals(index.getName()); int numberOfShardRouting = in.readVInt(); - logger.debug("Number of Index Routing Table {}", numberOfShardRouting); IndexRoutingTable.Builder indicesRoutingTable = IndexRoutingTable.builder(index); for (int idx = 0; idx < numberOfShardRouting; idx++) { IndexShardRoutingTable indexShardRoutingTable = IndexShardRoutingTable.Builder.readFrom(in); - logger.debug("Index Shard Routing Table reading {}", indexShardRoutingTable); indicesRoutingTable.addIndexShard(indexShardRoutingTable); - } verifyCheckSum(in); return indicesRoutingTable.build(); @@ -55,7 +69,7 @@ public IndexRoutingTable readIndexRoutingTable(Index index) throws IOException { } } - private void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException { + public static void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException { long expectedChecksum = in.getChecksum(); long readChecksum = in.readLong(); if (readChecksum != expectedChecksum) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java similarity index 62% rename from server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java rename to server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java index a37883ed7a5a5..23b853f488f46 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java @@ -19,25 +19,36 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; -public class IndexRoutingTableInputTests extends OpenSearchTestCase { +public class RemoteIndexRoutingTableObjectTests extends OpenSearchTestCase { public void testRoutingTableInput() { + int numberOfShards = randomIntBetween(1, 10); + int numberOfReplicas = randomIntBetween(1, 10); Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .put( + IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(numberOfShards) + .numberOfReplicas(numberOfReplicas) + ) .build(); RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - IndexRoutingTableInput indexRouting = new IndexRoutingTableInput(indexShardRoutingTables); - - IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRouting.write().streamInput()); - IndexRoutingTable indexRoutingTable = reader.readIndexRoutingTable(metadata.index("test").getIndex()); + RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); + IndexRoutingTable indexRoutingTable = RemoteIndexRoutingTableObject.read( + indexRouting.write().streamInput(), + metadata.index("test").getIndex() + ); - assertEquals(1, indexRoutingTable.getShards().size()); + assertEquals(numberOfShards, indexRoutingTable.getShards().size()); assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); - assertEquals(indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), 2); + assertEquals( + indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), + numberOfShards * (1 + numberOfReplicas) + ); } catch (IOException e) { throw new RuntimeException(e); } @@ -54,11 +65,8 @@ public void testRoutingTableInputStreamWithInvalidIndex() { AtomicInteger assertionError = new AtomicInteger(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - IndexRoutingTableInput indexRouting = new IndexRoutingTableInput(indexShardRoutingTables); - - IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRouting.write().streamInput()); - reader.readIndexRoutingTable(metadata.index("invalid-index").getIndex()); - + RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); + RemoteIndexRoutingTableObject.read(indexRouting.write().streamInput(), metadata.index("invalid-index").getIndex()); } catch (AssertionError e) { assertionError.getAndIncrement(); } catch (IOException e) { From a7f7cab814387e39e44b08055e58c6addd08b53c Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Wed, 29 May 2024 15:42:15 +0530 Subject: [PATCH 26/50] Add Manifest Tests and spotless fix Signed-off-by: Arpit Bandejiya --- .../stream/BufferedChecksumStreamOutput.java | 1 - .../remote/ClusterMetadataManifestTests.java | 83 ++++++++++++++++++- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 0f370f2d2f1fe..422f956c0cd47 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -34,7 +34,6 @@ import org.apache.lucene.store.BufferedChecksum; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.zip.CRC32; diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index e5fee8a5d7ad9..d1f559eb75f85 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -99,7 +99,7 @@ public void testClusterMetadataManifestXContent() throws IOException { Version.CURRENT, "test-node-id", false, - ClusterMetadataManifest.CODEC_V2, + ClusterMetadataManifest.CODEC_V3, null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", @@ -123,7 +123,9 @@ public void testClusterMetadataManifestXContent() throws IOException { "custom--weighted_routing_netadata-file" ) ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), + 1L, + randomUploadedIndexMetadataList() ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -169,7 +171,9 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "custom--weighted_routing_netadata-file" ) ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), + 1L, + randomUploadedIndexMetadataList() ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -311,6 +315,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { public void testClusterMetadataManifestXContentV2() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( 1L, 1L, @@ -320,10 +325,80 @@ public void testClusterMetadataManifestXContentV2() throws IOException { "test-node-id", false, ClusterMetadataManifest.CODEC_V2, - "test-metadata", + null, + Collections.singletonList(uploadedIndexMetadata), + "prev-cluster-uuid", + true, + uploadedMetadataAttribute, + uploadedMetadataAttribute, + uploadedMetadataAttribute, + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) + ) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), + 0, + new ArrayList<>() + ); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV2(parser); + assertEquals(originalManifest, fromXContentManifest); + } + } + + public void testClusterMetadataManifestXContentV3() throws IOException { + UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); + ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( + 1L, + 1L, + "test-cluster-uuid", + "test-state-uuid", + Version.CURRENT, + "test-node-id", + false, + ClusterMetadataManifest.CODEC_V3, + null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", true, + uploadedMetadataAttribute, + uploadedMetadataAttribute, + uploadedMetadataAttribute, + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) + ) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), 1L, Collections.singletonList(uploadedIndexMetadata) ); From 49bf7d02f02a42a176b800b9dd5681f4cd018e99 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 29 May 2024 17:53:25 +0530 Subject: [PATCH 27/50] Draft changes for remote routing table write Signed-off-by: Himshikha Gupta --- .../remote/RemoteRoutingTableService.java | 170 ++++++++++++++++++ .../remote/ClusterMetadataManifest.java | 32 +++- .../remote/RemoteClusterStateService.java | 59 +++++- .../routingtable/IndexRoutingTableInput.java | 3 + .../IndexRoutingTableInputStreamReader.java | 3 + .../index/remote/RemoteIndexPath.java | 4 +- .../index/remote/RemoteStoreEnums.java | 93 ++++++---- .../index/remote/RemoteStorePathStrategy.java | 80 +++++++-- .../RemoteSegmentStoreDirectoryFactory.java | 4 +- .../RemoteStoreLockManagerFactory.java | 2 +- .../index/translog/RemoteFsTranslog.java | 4 +- .../index/remote/RemoteStoreEnumsTests.java | 88 ++++----- .../opensearch/test/OpenSearchTestCase.java | 2 +- 13 files changed, 425 insertions(+), 119 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 5d6392be37ba6..53e3b00a75198 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,9 +10,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInput; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -23,6 +28,32 @@ import java.io.IOException; import java.util.function.Supplier; +import org.apache.lucene.store.IndexInput; +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; + +import org.opensearch.common.lucene.store.ByteArrayIndexInput; +import org.opensearch.core.action.ActionListener; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.index.remote.RemoteStorePathStrategy; +import org.opensearch.index.remote.RemoteStoreUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.opensearch.common.blobstore.transfer.RemoteTransferContainer.checksumOfChecksum; + import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; /** @@ -41,6 +72,12 @@ public class RemoteRoutingTableService implements Closeable { Setting.Property.NodeScope, Setting.Property.Final ); + + public static final String INDEX_ROUTING_PATH_TOKEN = "index-routing"; + public static final String INDEX_ROUTING_FILE_PREFIX = "index_routing"; + public static final String DELIMITER = "__"; + public static final String INDEX_ROUTING_METADATA_PREFIX = "indexRouting--"; + private static final Logger logger = LogManager.getLogger(RemoteRoutingTableService.class); private final Settings settings; private final Supplier repositoriesService; @@ -52,6 +89,139 @@ public RemoteRoutingTableService(Supplier repositoriesServi this.settings = settings; } + public List getChangedIndicesRouting(ClusterState previousClusterState, + ClusterState clusterState) { + Map previousIndexRoutingTable = previousClusterState.getRoutingTable().getIndicesRouting(); + List changedIndicesRouting = new ArrayList<>(); + for (IndexRoutingTable indexRouting : clusterState.getRoutingTable().getIndicesRouting().values()) { + if (!(previousIndexRoutingTable.containsKey(indexRouting.getIndex().getName()) && indexRouting.equals(previousIndexRoutingTable.get(indexRouting.getIndex().getName())))) { + changedIndicesRouting.add(indexRouting); + logger.info("changedIndicesRouting {}", indexRouting.prettyPrint()); + } + } + + return changedIndicesRouting; + } + + public CheckedRunnable getIndexRoutingAsyncAction( + ClusterState clusterState, + IndexRoutingTable indexRouting, + LatchedActionListener latchedActionListener, + BlobPath clusterBasePath + ) throws IOException { + + BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN); + logger.info("custerMetadataBasePath {}", indexRoutingPath); + + BlobPath path = RemoteStoreEnums.PathType.HASHED_PREFIX.path(RemoteStorePathStrategy.BasePathInput.builder() + .basePath(indexRoutingPath) + .indexUUID(indexRouting.getIndex().getUUID()) + .build(), + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64); + logger.info("path from prefix hasd {}", path); + final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); + + final String fileName = getIndexRoutingFileName(); + logger.info("fileName {}", fileName); + + ActionListener completionListener = ActionListener.wrap( + resp -> latchedActionListener.onResponse( + new ClusterMetadataManifest.UploadedIndexMetadata( + indexRouting.getIndex().getName(), + indexRouting.getIndex().getUUID(), + path.buildAsString() + fileName, + INDEX_ROUTING_METADATA_PREFIX + ) + ), + ex -> latchedActionListener.onFailure(new RemoteClusterStateService.RemoteStateTransferException(indexRouting.getIndex().toString(), ex)) + ); + + if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { + logger.info("TRYING FILE UPLOAD"); + + return () -> { + logger.info("Going to upload {}", indexRouting.prettyPrint()); + + uploadIndex(indexRouting, fileName , blobContainer); + logger.info("upload done {}", indexRouting.prettyPrint()); + + completionListener.onResponse(null); + logger.info("response done {}", indexRouting.prettyPrint()); + + }; + } + + logger.info("TRYING S3 UPLOAD"); + + IndexRoutingTableInput indexRoutingInput = new IndexRoutingTableInput(indexRouting); + BytesReference bytesInput = indexRoutingInput.write(); + try ( + IndexInput input = new ByteArrayIndexInput("indexrouting",BytesReference.toBytes(bytesInput))) { + long expectedChecksum; + try { + expectedChecksum = checksumOfChecksum(input.clone(), 8); + } catch (Exception e) { + throw e; + } + try ( + RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( + fileName, + fileName, + input.length(), + true, + WritePriority.URGENT, + (size, position) -> new OffsetRangeIndexInputStream(input, size, position), + expectedChecksum, + ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported() + ) + ) { + return () -> ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), completionListener); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + } + + + public List getAllUploadedIndicesRouting(ClusterMetadataManifest previousManifest, List indicesRoutingToUpload, Set indicesRoutingToDelete) { + final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() + .stream() + .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); + + indicesRoutingToUpload.forEach( + uploadedIndexRouting -> allUploadedIndicesRouting.put(uploadedIndexRouting.getIndexName(), uploadedIndexRouting) + ); + + indicesRoutingToDelete.forEach(index -> allUploadedIndicesRouting.remove(index)); + + logger.info("allUploadedIndicesRouting ROUTING {}", allUploadedIndicesRouting); + + return new ArrayList<>(allUploadedIndicesRouting.values()); + } + + private void uploadIndex(IndexRoutingTable indexRouting, String fileName, BlobContainer container) { + logger.info("Starting write"); + + try { + IndexRoutingTableInput indexRoutingInput = new IndexRoutingTableInput(indexRouting); + BytesReference bytesInput = indexRoutingInput.write(); + container.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); + logger.info("SUccessful write"); + } catch (IOException e) { + logger.error("Failed to write {}", e); + } + } + + private String getIndexRoutingFileName() { + return String.join( + DELIMITER, + INDEX_ROUTING_FILE_PREFIX, + RemoteStoreUtils.invertLong(System.currentTimeMillis()) + ); + + } + @Override public void close() throws IOException { if (blobStoreRepository != null) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 72e3826a3223a..9179d631d3409 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -807,6 +807,7 @@ public static class UploadedIndexMetadata implements UploadedMetadata, Writeable private static final ParseField INDEX_NAME_FIELD = new ParseField("index_name"); private static final ParseField INDEX_UUID_FIELD = new ParseField("index_uuid"); private static final ParseField UPLOADED_FILENAME_FIELD = new ParseField("uploaded_filename"); + private static final ParseField COMPONENT_PREFIX_FIELD = new ParseField("component_prefix"); private static String indexName(Object[] fields) { return (String) fields[0]; @@ -820,23 +821,34 @@ private static String uploadedFilename(Object[] fields) { return (String) fields[2]; } + private static String componentPrefix(Object[] fields) { + return (String) fields[3]; + } + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "uploaded_index_metadata", - fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields)) + fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), componentPrefix(fields)) ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD); } static final String COMPONENT_PREFIX = "index--"; + private final String componentPrefix; private final String indexName; private final String indexUUID; private final String uploadedFilename; public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName) { + this( indexName,indexUUID,uploadedFileName, COMPONENT_PREFIX); + } + + public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) { + this.componentPrefix = componentPrefix; this.indexName = indexName; this.indexUUID = indexUUID; this.uploadedFilename = uploadedFileName; @@ -846,6 +858,7 @@ public UploadedIndexMetadata(StreamInput in) throws IOException { this.indexName = in.readString(); this.indexUUID = in.readString(); this.uploadedFilename = in.readString(); + this.componentPrefix = in.readString(); } public String getUploadedFilePath() { @@ -854,7 +867,7 @@ public String getUploadedFilePath() { @Override public String getComponent() { - return COMPONENT_PREFIX + getIndexName(); + return componentPrefix + getIndexName(); } public String getUploadedFilename() { @@ -870,11 +883,18 @@ public String getIndexUUID() { return indexUUID; } + public String getComponentPrefix() { + return componentPrefix; + } + + + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) .field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID()) - .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()); + .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()) + .field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix()); } @Override @@ -882,6 +902,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(indexName); out.writeString(indexUUID); out.writeString(uploadedFilename); + out.writeString(componentPrefix); } @Override @@ -895,12 +916,13 @@ public boolean equals(Object o) { final UploadedIndexMetadata that = (UploadedIndexMetadata) o; return Objects.equals(indexName, that.indexName) && Objects.equals(indexUUID, that.indexUUID) - && Objects.equals(uploadedFilename, that.uploadedFilename); + && Objects.equals(uploadedFilename, that.uploadedFilename) + && Objects.equals(componentPrefix, that.componentPrefix); } @Override public int hashCode() { - return Objects.hash(indexName, indexUUID, uploadedFilename); + return Objects.hash(indexName, indexUUID, uploadedFilename, componentPrefix); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 34572f1d9009f..97a388882599c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -18,6 +18,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -218,7 +219,7 @@ public class RemoteClusterStateService implements Closeable { + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + "updated : [{}], custom metadata updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2; // ToXContent Params with gateway mode. @@ -289,7 +290,8 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri clusterState.metadata().customs(), true, true, - true + true, + new ArrayList<>(clusterState.getRoutingTable().indicesRouting().values()) ); final ClusterMetadataManifest manifest = uploadManifest( clusterState, @@ -299,7 +301,8 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedSettingsMetadata, uploadedMetadataResults.uploadedTemplatesMetadata, uploadedMetadataResults.uploadedCustomMetadataMap, - false + uploadedMetadataResults.uploadedIndicesRoutingMetadata, + false ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); @@ -380,6 +383,12 @@ public ClusterMetadataManifest writeIncrementalMetadata( // index present in current cluster state indicesToBeDeletedFromRemote.remove(indexMetadata.getIndex().getName()); } + + List indicesRoutingToUpload = new ArrayList<>(); + if(remoteRoutingTableService.isPresent()) { + indicesRoutingToUpload = remoteRoutingTableService.get().getChangedIndicesRouting(previousClusterState, clusterState); + } + UploadedMetadataResults uploadedMetadataResults; // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, // If file is empty and codec is 1 then write global metadata. @@ -399,7 +408,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( firstUploadForSplitGlobalMetadata ? clusterState.metadata().customs() : customsToUpload, updateCoordinationMetadata, updateSettingsMetadata, - updateTemplatesMetadata + updateTemplatesMetadata, + indicesRoutingToUpload ); // update the map if the metadata was uploaded @@ -411,6 +421,11 @@ public ClusterMetadataManifest writeIncrementalMetadata( customsToBeDeletedFromRemote.keySet().forEach(allUploadedCustomMap::remove); indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove); + List allUploadedIndicesRouting = new ArrayList<>(); + if(remoteRoutingTableService.isPresent()) { + allUploadedIndicesRouting = remoteRoutingTableService.get().getAllUploadedIndicesRouting(previousManifest, uploadedMetadataResults.uploadedIndicesRoutingMetadata, indicesToBeDeletedFromRemote.keySet()); + } + final ClusterMetadataManifest manifest = uploadManifest( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), @@ -421,10 +436,13 @@ public ClusterMetadataManifest writeIncrementalMetadata( firstUploadForSplitGlobalMetadata || !customsToUpload.isEmpty() ? allUploadedCustomMap : previousManifest.getCustomMetadataMap(), + allUploadedIndicesRouting, false ); deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS); + logger.info("MANIFEST IN INC STATE {}", manifest); + final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); remoteStateStats.stateTook(durationMillis); @@ -462,11 +480,12 @@ private UploadedMetadataResults writeMetadataInParallel( Map customToUpload, boolean uploadCoordinationMetadata, boolean uploadSettingsMetadata, - boolean uploadTemplateMetadata + boolean uploadTemplateMetadata, + List indicesRoutingToUpload ) throws IOException { assert Objects.nonNull(indexMetadataUploadListeners) : "indexMetadataUploadListeners can not be null"; int totalUploadTasks = indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size() - + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0); + + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + indicesRoutingToUpload.size(); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new HashMap<>(totalUploadTasks); Map results = new HashMap<>(totalUploadTasks); @@ -533,6 +552,18 @@ private UploadedMetadataResults writeMetadataInParallel( uploadTasks.put(indexMetadata.getIndex().getName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); }); + indicesRoutingToUpload.forEach(indexRoutingTable -> { + try { + uploadTasks.put( + indexRoutingTable.getIndex().getName() + "--indexRouting", + remoteRoutingTableService.get().getIndexRoutingAsyncAction(clusterState, indexRoutingTable, listener, getCusterMetadataBasePath(clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID())) + ); + } catch (IOException e) { + e.printStackTrace(); + } + }); + // start async upload of all required metadata files for (CheckedRunnable uploadTask : uploadTasks.values()) { uploadTask.run(); @@ -579,7 +610,10 @@ private UploadedMetadataResults writeMetadataInParallel( } UploadedMetadataResults response = new UploadedMetadataResults(); results.forEach((name, uploadedMetadata) -> { - if (name.contains(CUSTOM_METADATA)) { + if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class) && + uploadedMetadata.getComponent().contains(RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) { + response.uploadedIndicesRoutingMetadata.add((UploadedIndexMetadata) uploadedMetadata); + } else if (name.contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom-- String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1]; response.uploadedCustomMetadataMap.put( @@ -598,6 +632,7 @@ private UploadedMetadataResults writeMetadataInParallel( throw new IllegalStateException("Unknown metadata component name " + name); } }); + logger.info("response {}", response.uploadedIndicesRoutingMetadata.toString()); return response; } @@ -744,6 +779,7 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat previousManifest.getSettingsMetadata(), previousManifest.getTemplatesMetadata(), previousManifest.getCustomMetadataMap(), + previousManifest.getIndicesRouting(), true ); deleteStaleClusterUUIDs(clusterState, committedManifest); @@ -782,6 +818,7 @@ private ClusterMetadataManifest uploadManifest( UploadedMetadataAttribute uploadedSettingsMetadata, UploadedMetadataAttribute uploadedTemplatesMetadata, Map uploadedCustomMetadataMap, + List uploadedIndicesRouting, boolean committed ) throws IOException { synchronized (this) { @@ -809,7 +846,7 @@ private ClusterMetadataManifest uploadManifest( uploadedTemplatesMetadata, uploadedCustomMetadataMap, clusterState.routingTable().version(), - new ArrayList<>() + uploadedIndicesRouting ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; @@ -1743,19 +1780,22 @@ private static class UploadedMetadataResults { UploadedMetadataAttribute uploadedCoordinationMetadata; UploadedMetadataAttribute uploadedSettingsMetadata; UploadedMetadataAttribute uploadedTemplatesMetadata; + List uploadedIndicesRoutingMetadata; public UploadedMetadataResults( List uploadedIndexMetadata, Map uploadedCustomMetadataMap, UploadedMetadataAttribute uploadedCoordinationMetadata, UploadedMetadataAttribute uploadedSettingsMetadata, - UploadedMetadataAttribute uploadedTemplatesMetadata + UploadedMetadataAttribute uploadedTemplatesMetadata, + List uploadedIndicesRoutingMetadata ) { this.uploadedIndexMetadata = uploadedIndexMetadata; this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; this.uploadedSettingsMetadata = uploadedSettingsMetadata; this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; + this.uploadedIndicesRoutingMetadata = uploadedIndicesRoutingMetadata; } public UploadedMetadataResults() { @@ -1764,6 +1804,7 @@ public UploadedMetadataResults() { this.uploadedCoordinationMetadata = null; this.uploadedSettingsMetadata = null; this.uploadedTemplatesMetadata = null; + this.uploadedIndicesRoutingMetadata = new ArrayList<>(); } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java index 3790351e5f7fa..eea9b8640b47c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java @@ -17,6 +17,9 @@ import java.io.IOException; import java.util.Iterator; +/** + * Class for creating object for uploading IndexRoutingTable to remote store. + */ public class IndexRoutingTableInput { private final IndexRoutingTableHeader indexRoutingTableHeader; diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java index d1c3d22a2cc34..64671511bd826 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.io.InputStream; +/** + * Reads IndexRoutingTable from S3* + */ public class IndexRoutingTableInputStreamReader { private final StreamInput streamInput; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java index 89b642b79df86..3629fcfa79420 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java @@ -18,7 +18,7 @@ import org.opensearch.index.remote.RemoteStoreEnums.DataType; import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; -import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; +import org.opensearch.index.remote.RemoteStorePathStrategy.RemoteStorePathInput; import java.io.IOException; import java.util.Collections; @@ -141,7 +141,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws DataCategory dataCategory = entry.getKey(); for (DataType type : entry.getValue()) { for (int shardNo = 0; shardNo < shardCount; shardNo++) { - PathInput pathInput = PathInput.builder() + RemoteStorePathInput pathInput = RemoteStorePathInput.builder() .basePath(new BlobPath().add(basePath)) .indexUUID(indexUUID) .shardId(Integer.toString(shardNo)) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java index c1ac74724e405..c7fb5f1e27ccc 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java @@ -13,7 +13,7 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.hash.FNV1a; -import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; +import org.opensearch.index.remote.RemoteStorePathStrategy.RemoteStorePathInput; import java.util.HashMap; import java.util.Locale; @@ -92,14 +92,18 @@ public String getName() { public enum PathType { FIXED(0) { @Override - public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { + public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.isNull(hashAlgorithm) : "hashAlgorithm is expected to be null with fixed remote store path type"; // Hash algorithm is not used in FIXED path type - return pathInput.basePath() - .add(pathInput.indexUUID()) - .add(pathInput.shardId()) - .add(pathInput.dataCategory().getName()) - .add(pathInput.dataType().getName()); + BlobPath path = pathInput.basePath() + .add(pathInput.indexUUID()); + if(pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput)pathInput; + path.add(remoteStorePathInput.shardId()) + .add(remoteStorePathInput.dataCategory().getName()) + .add(remoteStorePathInput.dataType().getName()); + } + return path; } @Override @@ -109,15 +113,19 @@ boolean requiresHashAlgorithm() { }, HASHED_PREFIX(1) { @Override - public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { + public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - return BlobPath.cleanPath() + BlobPath path = BlobPath.cleanPath() .add(hashAlgorithm.hash(pathInput)) .add(pathInput.basePath()) - .add(pathInput.indexUUID()) - .add(pathInput.shardId()) - .add(pathInput.dataCategory().getName()) - .add(pathInput.dataType().getName()); + .add(pathInput.indexUUID()); + if(pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput)pathInput; + path.add(remoteStorePathInput.shardId()) + .add(remoteStorePathInput.dataCategory().getName()) + .add(remoteStorePathInput.dataType().getName()); + } + return path; } @Override @@ -127,14 +135,18 @@ boolean requiresHashAlgorithm() { }, HASHED_INFIX(2) { @Override - public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { + public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - return pathInput.basePath() + BlobPath path = pathInput.basePath() .add(hashAlgorithm.hash(pathInput)) - .add(pathInput.indexUUID()) - .add(pathInput.shardId()) - .add(pathInput.dataCategory().getName()) - .add(pathInput.dataType().getName()); + .add(pathInput.indexUUID()); + if(pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput)pathInput; + path.add(remoteStorePathInput.shardId()) + .add(remoteStorePathInput.dataCategory().getName()) + .add(remoteStorePathInput.dataType().getName()); + } + return path; } @Override @@ -185,18 +197,21 @@ public static PathType fromCode(int code) { * @param hashAlgorithm hashing algorithm. * @return the blob path for the path input. */ - public BlobPath path(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { - DataCategory dataCategory = pathInput.dataCategory(); - DataType dataType = pathInput.dataType(); - assert dataCategory.isSupportedDataType(dataType) : "category:" - + dataCategory - + " type:" - + dataType - + " are not supported together"; + public BlobPath path(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { + if(pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput)pathInput; + DataCategory dataCategory = remoteStorePathInput.dataCategory(); + DataType dataType = remoteStorePathInput.dataType(); + assert dataCategory.isSupportedDataType(dataType) : "category:" + + dataCategory + + " type:" + + dataType + + " are not supported together"; + } return generatePath(pathInput, hashAlgorithm); } - protected abstract BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm); + protected abstract BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm); abstract boolean requiresHashAlgorithm(); @@ -226,9 +241,13 @@ public enum PathHashAlgorithm { FNV_1A_BASE64(0) { @Override - String hash(PathInput pathInput) { - String input = pathInput.indexUUID() + pathInput.shardId() + pathInput.dataCategory().getName() + pathInput.dataType() - .getName(); + String hash(RemoteStorePathStrategy.BasePathInput pathInput) { + String input = pathInput.indexUUID(); + if(pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; + input += remoteStorePathInput.shardId() + remoteStorePathInput.dataCategory().getName() + remoteStorePathInput.dataType() + .getName(); + } long hash = FNV1a.hash64(input); return longToUrlBase64(hash); } @@ -239,9 +258,13 @@ String hash(PathInput pathInput) { */ FNV_1A_COMPOSITE_1(1) { @Override - String hash(PathInput pathInput) { - String input = pathInput.indexUUID() + pathInput.shardId() + pathInput.dataCategory().getName() + pathInput.dataType() - .getName(); + String hash(RemoteStorePathStrategy.BasePathInput pathInput) { + String input = pathInput.indexUUID(); + if(pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; + input += remoteStorePathInput.shardId() + remoteStorePathInput.dataCategory().getName() + remoteStorePathInput.dataType() + .getName(); + } long hash = FNV1a.hash64(input); return longToCompositeBase64AndBinaryEncoding(hash, 20); } @@ -281,7 +304,7 @@ public static PathHashAlgorithm fromCode(int code) { return CODE_TO_ENUM.get(code); } - abstract String hash(PathInput pathInput); + abstract String hash(RemoteStorePathStrategy.BasePathInput pathInput); public static PathHashAlgorithm parseString(String pathHashAlgorithm) { try { diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java index c58f6c3faac84..252fcde2c4105 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java @@ -67,7 +67,7 @@ public String toString() { return "RemoteStorePathStrategy{" + "type=" + type + ", hashAlgorithm=" + hashAlgorithm + '}'; } - public BlobPath generatePath(PathInput pathInput) { + public BlobPath generatePath(RemoteStorePathInput pathInput) { return type.path(pathInput, hashAlgorithm); } @@ -77,19 +77,13 @@ public BlobPath generatePath(PathInput pathInput) { */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class PathInput { + public static class BasePathInput { private final BlobPath basePath; private final String indexUUID; - private final String shardId; - private final DataCategory dataCategory; - private final DataType dataType; - public PathInput(BlobPath basePath, String indexUUID, String shardId, DataCategory dataCategory, DataType dataType) { + public BasePathInput(BlobPath basePath, String indexUUID) { this.basePath = Objects.requireNonNull(basePath); this.indexUUID = Objects.requireNonNull(indexUUID); - this.shardId = Objects.requireNonNull(shardId); - this.dataCategory = Objects.requireNonNull(dataCategory); - this.dataType = Objects.requireNonNull(dataType); } BlobPath basePath() { @@ -100,6 +94,58 @@ String indexUUID() { return indexUUID; } + /** + * Returns a new builder for {@link BasePathInput}. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for {@link BasePathInput}. + * + * @opensearch.internal + */ + @PublicApi(since = "2.14.0") + @ExperimentalApi + public static class Builder { + private BlobPath basePath; + private String indexUUID; + + public Builder basePath(BlobPath basePath) { + this.basePath = basePath; + return this; + } + + public Builder indexUUID(String indexUUID) { + this.indexUUID = indexUUID; + return this; + } + + public BasePathInput build() { + return new BasePathInput(basePath, indexUUID); + } + } + } + + /** + * Wrapper class for the input required to generate path for remote store uploads. + * @opensearch.internal + */ + @PublicApi(since = "2.14.0") + @ExperimentalApi + public static class RemoteStorePathInput extends BasePathInput{ + private final String shardId; + private final DataCategory dataCategory; + private final DataType dataType; + + public RemoteStorePathInput(BlobPath basePath, String indexUUID, String shardId, DataCategory dataCategory, DataType dataType) { + super(basePath, indexUUID); + this.shardId = Objects.requireNonNull(shardId); + this.dataCategory = Objects.requireNonNull(dataCategory); + this.dataType = Objects.requireNonNull(dataType); + } + String shardId() { return shardId; } @@ -113,33 +159,31 @@ DataType dataType() { } /** - * Returns a new builder for {@link PathInput}. + * Returns a new builder for {@link RemoteStorePathInput}. */ public static Builder builder() { return new Builder(); } /** - * Builder for {@link PathInput}. + * Builder for {@link RemoteStorePathInput}. * * @opensearch.internal */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class Builder { - private BlobPath basePath; - private String indexUUID; + public static class Builder extends BasePathInput.Builder{ private String shardId; private DataCategory dataCategory; private DataType dataType; public Builder basePath(BlobPath basePath) { - this.basePath = basePath; + super.basePath = basePath; return this; } public Builder indexUUID(String indexUUID) { - this.indexUUID = indexUUID; + super.indexUUID = indexUUID; return this; } @@ -158,8 +202,8 @@ public Builder dataType(DataType dataType) { return this; } - public PathInput build() { - return new PathInput(basePath, indexUUID, shardId, dataCategory, dataType); + public RemoteStorePathInput build() { + return new RemoteStorePathInput(super.basePath, super.indexUUID, shardId, dataCategory, dataType); } } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index e462f6d4ac011..72def0f2b2233 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -65,7 +65,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s BlobPath repositoryBasePath = blobStoreRepository.basePath(); String shardIdStr = String.valueOf(shardId.id()); - RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.RemoteStorePathInput dataPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardIdStr) @@ -80,7 +80,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s blobStoreRepository::maybeRateLimitRemoteDownloadTransfers ); - RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.RemoteStorePathInput mdPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardIdStr) diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java index 45d466d3a8ce8..9fc118699caa5 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java @@ -56,7 +56,7 @@ public static RemoteStoreMetadataLockManager newLockManager( assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobPath repositoryBasePath = ((BlobStoreRepository) repository).basePath(); - RemoteStorePathStrategy.PathInput lockFilesPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.RemoteStorePathInput lockFilesPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardId) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 67549c86b7dd2..41b3ad5cc378f 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -304,7 +304,7 @@ public static TranslogTransferManager buildTranslogTransferManager( assert Objects.nonNull(pathStrategy); String indexUUID = shardId.getIndex().getUUID(); String shardIdStr = String.valueOf(shardId.id()); - RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.RemoteStorePathInput dataPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() .basePath(blobStoreRepository.basePath()) .indexUUID(indexUUID) .shardId(shardIdStr) @@ -312,7 +312,7 @@ public static TranslogTransferManager buildTranslogTransferManager( .dataType(DATA) .build(); BlobPath dataPath = pathStrategy.generatePath(dataPathInput); - RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.RemoteStorePathInput mdPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() .basePath(blobStoreRepository.basePath()) .indexUUID(indexUUID) .shardId(shardIdStr) diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java index 575b397382f24..f6999cf71c9e6 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java @@ -13,7 +13,7 @@ import org.opensearch.index.remote.RemoteStoreEnums.DataCategory; import org.opensearch.index.remote.RemoteStoreEnums.DataType; import org.opensearch.index.remote.RemoteStoreEnums.PathType; -import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; +import org.opensearch.index.remote.RemoteStorePathStrategy.RemoteStorePathInput; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; @@ -70,7 +70,7 @@ public void testGeneratePathForFixedType() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId + SEPARATOR; // Translog Data - PathInput pathInput = PathInput.builder() + RemoteStorePathInput pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -82,7 +82,7 @@ public void testGeneratePathForFixedType() { // Translog Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -94,7 +94,7 @@ public void testGeneratePathForFixedType() { // Translog Lock files - This is a negative case where the assertion will trip. dataType = LOCK_FILES; - PathInput finalPathInput = PathInput.builder() + RemoteStorePathInput finalPathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -106,7 +106,7 @@ public void testGeneratePathForFixedType() { // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -118,7 +118,7 @@ public void testGeneratePathForFixedType() { // Segment Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -130,7 +130,7 @@ public void testGeneratePathForFixedType() { // Segment Metadata dataType = LOCK_FILES; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -155,7 +155,7 @@ public void testGeneratePathForHashedPrefixType() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId; // Translog Data - PathInput pathInput = PathInput.builder() + RemoteStorePathInput pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -172,7 +172,7 @@ public void testGeneratePathForHashedPrefixType() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -184,7 +184,7 @@ public void testGeneratePathForHashedPrefixType() { // Translog Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -198,7 +198,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -210,7 +210,7 @@ public void testGeneratePathForHashedPrefixType() { // Translog Lock files - This is a negative case where the assertion will trip. dataType = LOCK_FILES; - PathInput finalPathInput = PathInput.builder() + RemoteStorePathInput finalPathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -220,7 +220,7 @@ public void testGeneratePathForHashedPrefixType() { assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -232,7 +232,7 @@ public void testGeneratePathForHashedPrefixType() { // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -246,7 +246,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -258,7 +258,7 @@ public void testGeneratePathForHashedPrefixType() { // Segment Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -272,7 +272,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -284,7 +284,7 @@ public void testGeneratePathForHashedPrefixType() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -298,7 +298,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -323,7 +323,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId; // Translog Data - PathInput pathInput = PathInput.builder() + RemoteStorePathInput pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -342,7 +342,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -354,7 +354,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Translog Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -370,7 +370,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -385,7 +385,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Translog Lock files - This is a negative case where the assertion will trip. dataType = LOCK_FILES; - PathInput finalPathInput = PathInput.builder() + RemoteStorePathInput finalPathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -395,7 +395,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -407,7 +407,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -423,7 +423,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -435,7 +435,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Segment Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -451,7 +451,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -466,7 +466,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -482,7 +482,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -511,7 +511,7 @@ public void testGeneratePathForHashedInfixType() { String basePath = getPath(pathList); basePath = basePath.length() == 0 ? basePath : basePath.substring(0, basePath.length() - 1); // Translog Data - PathInput pathInput = PathInput.builder() + RemoteStorePathInput pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -527,7 +527,7 @@ public void testGeneratePathForHashedInfixType() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -541,7 +541,7 @@ public void testGeneratePathForHashedInfixType() { // Translog Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -555,7 +555,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -569,7 +569,7 @@ public void testGeneratePathForHashedInfixType() { // Translog Lock files - This is a negative case where the assertion will trip. dataType = LOCK_FILES; - PathInput finalPathInput = PathInput.builder() + RemoteStorePathInput finalPathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -579,7 +579,7 @@ public void testGeneratePathForHashedInfixType() { assertThrows(AssertionError.class, () -> HASHED_INFIX.path(finalPathInput, null)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -591,7 +591,7 @@ public void testGeneratePathForHashedInfixType() { // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -604,7 +604,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -618,7 +618,7 @@ public void testGeneratePathForHashedInfixType() { // Segment Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -631,7 +631,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -645,7 +645,7 @@ public void testGeneratePathForHashedInfixType() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -658,7 +658,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = RemoteStorePathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -671,7 +671,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); } - private String derivePath(String basePath, PathInput pathInput) { + private String derivePath(String basePath, RemoteStorePathInput pathInput) { return "".equals(basePath) ? String.join( SEPARATOR, diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 5a3f3b5a07a8d..445618228cc15 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -1828,7 +1828,7 @@ public static BlobPath getShardLevelBlobPath( ? RemoteStoreEnums.PathHashAlgorithm.valueOf(remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME)) : null : null; - RemoteStorePathStrategy.PathInput pathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.RemoteStorePathInput pathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() .basePath(basePath) .indexUUID(indexUUID) .shardId(shardId) From 71e151f6570e73cdd1d2d5a68a3ee25dedaaf264 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Thu, 30 May 2024 12:38:35 +0530 Subject: [PATCH 28/50] Fix remoteClusterServiceTests Signed-off-by: Arpit Bandejiya --- .../gateway/remote/RemoteClusterStateService.java | 2 +- .../gateway/remote/RemoteClusterStateServiceTests.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 9ce7d67a5706b..9e9caf47a0997 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -215,7 +215,7 @@ public class RemoteClusterStateService implements Closeable { + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + "updated : [{}], custom metadata updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2; // ToXContent Params with gateway mode. diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index d95d95fba630f..e64865da69b65 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -547,7 +547,7 @@ private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMe ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() - .codecVersion(2) + .codecVersion(3) .indices(Collections.emptyList()) .clusterTerm(1L) .stateVersion(1L) @@ -1559,7 +1559,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap1 = Map.of("index-uuid1", indexMetadata1, "index-uuid2", indexMetadata2); mockBlobContainerForGlobalMetadata(blobContainer1, clusterManifest1, metadata1); - mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V2); + mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V3); List uploadedIndexMetadataList2 = List.of( new UploadedIndexMetadata("index1", "index-uuid1", "key1"), @@ -1591,7 +1591,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap2 = Map.of("index-uuid1", indexMetadata3, "index-uuid2", indexMetadata4); mockBlobContainerForGlobalMetadata(blobContainer2, clusterManifest2, metadata2); - mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V2); + mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V3); // differGlobalMetadata controls which one of IndexMetadata or Metadata object would be different // when comparing cluster-uuid3 and cluster-uuid1 state. @@ -1625,7 +1625,7 @@ private void mockObjectsForGettingPreviousClusterUUID( clusterUUIDCommitted.getOrDefault("cluster-uuid3", true) ); mockBlobContainerForGlobalMetadata(blobContainer3, clusterManifest3, metadata3); - mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V2); + mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V3); ArrayList mockBlobContainerOrderedList = new ArrayList<>( List.of(blobContainer1, blobContainer1, blobContainer3, blobContainer3, blobContainer2, blobContainer2) From a9ab9c10783656ff5bb4ecfa155f16c36319e6ed Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 31 May 2024 15:17:49 +0530 Subject: [PATCH 29/50] Remote store objects to implement writeable interface Signed-off-by: Himshikha Gupta --- .../routingtable/IndexRoutingTableHeader.java | 32 +++++----- .../RemoteIndexRoutingTableObject.java | 60 ++++++++++--------- .../IndexRoutingTableHeaderTests.java | 4 +- .../RemoteIndexRoutingTableObjectTests.java | 16 +++-- 4 files changed, 63 insertions(+), 49 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index bc99fea9b5c09..059b385394cb5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -16,6 +16,7 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.EOFException; import java.io.IOException; @@ -23,7 +24,7 @@ /** * The stored header information for the individual index routing table */ -public class IndexRoutingTableHeader { +public class IndexRoutingTableHeader implements Writeable { public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; public static final int INITIAL_VERSION = 1; @@ -35,41 +36,42 @@ public IndexRoutingTableHeader(String indexName) { } /** - * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} + * Reads the contents on the stream into the corresponding {@link IndexRoutingTableHeader} * * @param in - * @return IndexRoutingTableHeader * @throws IOException */ - public static IndexRoutingTableHeader read(StreamInput in) throws IOException { + public IndexRoutingTableHeader(StreamInput in) throws IOException { try { readHeaderVersion(in); - final String name = in.readString(); - return new IndexRoutingTableHeader(name); + indexName = in.readString(); } catch (EOFException e) { throw new IOException("index routing header truncated", e); } } - static int readHeaderVersion(final StreamInput in) throws IOException { - final int version; + private void readHeaderVersion(final StreamInput in) throws IOException { try { - version = CodecUtil.checkHeader(new InputStreamDataInput(in), INDEX_ROUTING_HEADER_CODEC, INITIAL_VERSION, CURRENT_VERSION); + CodecUtil.checkHeader(new InputStreamDataInput(in), INDEX_ROUTING_HEADER_CODEC, INITIAL_VERSION, CURRENT_VERSION); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { throw new IOException("index routing table header corrupted", e); } - return version; } /** - * Returns the bytes reference for the {@link IndexRoutingTableHeader} + * Write the IndexRoutingTable to given stream. * + * @param out stream to write * @throws IOException */ - public void write(StreamOutput out) throws IOException { - CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); - out.writeString(indexName); - out.flush(); + public void writeTo(StreamOutput out) throws IOException { + try { + CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); + out.writeString(indexName); + out.flush(); + } catch (IOException e) { + throw new IOException("Failed to write IndexRoutingTable header", e); + } } public String getIndexName() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java index 263fb29120553..91ab7ecf413c0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java @@ -10,49 +10,33 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.Index; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; -import java.util.Iterator; -public class RemoteIndexRoutingTableObject { +/** + * Remote store object for IndexRoutingTable + */ +public class RemoteIndexRoutingTableObject implements Writeable { - private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; - private final int shardCount; + private final IndexRoutingTable indexRoutingTable; public RemoteIndexRoutingTableObject(IndexRoutingTable indexRoutingTable) { - this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); - this.shardCount = indexRoutingTable.shards().size(); - } - - public BytesReference write() throws IOException { - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput); - indexRoutingTableHeader.write(out); - out.writeVInt(shardCount); - while (shardIter.hasNext()) { - IndexShardRoutingTable next = shardIter.next(); - IndexShardRoutingTable.Builder.writeTo(next, out); - } - out.writeLong(out.getChecksum()); - out.flush(); - return bytesStreamOutput.bytes(); + this.indexRoutingTable = indexRoutingTable; } - public static IndexRoutingTable read(InputStream inputStream, Index index) throws IOException { + public RemoteIndexRoutingTableObject(InputStream inputStream, Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) { // Read the Table Header first and confirm the index - IndexRoutingTableHeader indexRoutingTableHeader = IndexRoutingTableHeader.read(in); + IndexRoutingTableHeader indexRoutingTableHeader = new IndexRoutingTableHeader(in); assert indexRoutingTableHeader.getIndexName().equals(index.getName()); int numberOfShardRouting = in.readVInt(); @@ -62,14 +46,34 @@ public static IndexRoutingTable read(InputStream inputStream, Index index) throw indicesRoutingTable.addIndexShard(indexShardRoutingTable); } verifyCheckSum(in); - return indicesRoutingTable.build(); + indexRoutingTable = indicesRoutingTable.build(); } } catch (EOFException e) { throw new IOException("Indices Routing table is corrupted", e); } } - public static void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException { + public IndexRoutingTable getIndexRoutingTable() { + return indexRoutingTable; + } + + public void writeTo(StreamOutput streamOutput) throws IOException { + try { + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(streamOutput); + IndexRoutingTableHeader indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); + indexRoutingTableHeader.writeTo(out); + out.writeVInt(indexRoutingTable.shards().size()); + for (IndexShardRoutingTable next : indexRoutingTable) { + IndexShardRoutingTable.Builder.writeTo(next, out); + } + out.writeLong(out.getChecksum()); + out.flush(); + } catch (IOException e) { + throw new IOException("Failed to write IndexRoutingTable to stream", e); + } + } + + private void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException { long expectedChecksum = in.getChecksum(); long readChecksum = in.readLong(); if (readChecksum != expectedChecksum) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index 0f42508615a26..187ef2b928f75 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -19,10 +19,10 @@ public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { public void testIndexRoutingTableHeader() throws IOException { IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); BytesStreamOutput out = new BytesStreamOutput(); - header.write(out); + header.writeTo(out); BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); - IndexRoutingTableHeader headerRead = IndexRoutingTableHeader.read(in); + IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in); assertEquals("dummyIndex", headerRead.getIndexName()); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java index 23b853f488f46..d4d01ea04b281 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java @@ -14,6 +14,7 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -38,11 +39,13 @@ public void testRoutingTableInput() { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); - IndexRoutingTable indexRoutingTable = RemoteIndexRoutingTableObject.read( - indexRouting.write().streamInput(), + BytesStreamOutput streamOutput = new BytesStreamOutput(); + indexRouting.writeTo(streamOutput); + RemoteIndexRoutingTableObject remoteIndexRoutingTable = new RemoteIndexRoutingTableObject( + streamOutput.bytes().streamInput(), metadata.index("test").getIndex() ); - + IndexRoutingTable indexRoutingTable = remoteIndexRoutingTable.getIndexRoutingTable(); assertEquals(numberOfShards, indexRoutingTable.getShards().size()); assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); assertEquals( @@ -66,7 +69,12 @@ public void testRoutingTableInputStreamWithInvalidIndex() { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); - RemoteIndexRoutingTableObject.read(indexRouting.write().streamInput(), metadata.index("invalid-index").getIndex()); + BytesStreamOutput streamOutput = new BytesStreamOutput(); + indexRouting.writeTo(streamOutput); + RemoteIndexRoutingTableObject remoteIndexRoutingTable = new RemoteIndexRoutingTableObject( + streamOutput.bytes().streamInput(), + metadata.index("invalid-index").getIndex() + ); } catch (AssertionError e) { assertionError.getAndIncrement(); } catch (IOException e) { From 4167eebf27f4d69fbf350ea06aa6eb0eadb6c851 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 3 Jun 2024 15:19:52 +0530 Subject: [PATCH 30/50] addressing pr comments Signed-off-by: Himshikha Gupta --- ...bleObject.java => RemoteIndexRoutingTable.java} | 7 ++++--- .../routingtable/IndexRoutingTableHeaderTests.java | 12 +++++++----- .../RemoteIndexRoutingTableObjectTests.java | 14 ++++++-------- 3 files changed, 17 insertions(+), 16 deletions(-) rename server/src/main/java/org/opensearch/gateway/remote/routingtable/{RemoteIndexRoutingTableObject.java => RemoteIndexRoutingTable.java} (93%) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java similarity index 93% rename from server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java rename to server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java index 91ab7ecf413c0..5641ef01df8d2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java @@ -24,15 +24,15 @@ /** * Remote store object for IndexRoutingTable */ -public class RemoteIndexRoutingTableObject implements Writeable { +public class RemoteIndexRoutingTable implements Writeable { private final IndexRoutingTable indexRoutingTable; - public RemoteIndexRoutingTableObject(IndexRoutingTable indexRoutingTable) { + public RemoteIndexRoutingTable(IndexRoutingTable indexRoutingTable) { this.indexRoutingTable = indexRoutingTable; } - public RemoteIndexRoutingTableObject(InputStream inputStream, Index index) throws IOException { + public RemoteIndexRoutingTable(InputStream inputStream, Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) { // Read the Table Header first and confirm the index @@ -57,6 +57,7 @@ public IndexRoutingTable getIndexRoutingTable() { return indexRoutingTable; } + @Override public void writeTo(StreamOutput streamOutput) throws IOException { try { BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(streamOutput); diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index 187ef2b928f75..bea65936fc32e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -18,12 +18,14 @@ public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { public void testIndexRoutingTableHeader() throws IOException { IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); - BytesStreamOutput out = new BytesStreamOutput(); - header.writeTo(out); + try (BytesStreamOutput out = new BytesStreamOutput()) { + header.writeTo(out); - BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); - IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in); - assertEquals("dummyIndex", headerRead.getIndexName()); + BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); + IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in); + assertEquals("dummyIndex", headerRead.getIndexName()); + + } } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java index d4d01ea04b281..d6fed51f692a5 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java @@ -37,11 +37,10 @@ public void testRoutingTableInput() { RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { - try { - RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); - BytesStreamOutput streamOutput = new BytesStreamOutput(); + RemoteIndexRoutingTable indexRouting = new RemoteIndexRoutingTable(indexShardRoutingTables); + try (BytesStreamOutput streamOutput = new BytesStreamOutput();) { indexRouting.writeTo(streamOutput); - RemoteIndexRoutingTableObject remoteIndexRoutingTable = new RemoteIndexRoutingTableObject( + RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable( streamOutput.bytes().streamInput(), metadata.index("test").getIndex() ); @@ -67,11 +66,10 @@ public void testRoutingTableInputStreamWithInvalidIndex() { RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); AtomicInteger assertionError = new AtomicInteger(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { - try { - RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); - BytesStreamOutput streamOutput = new BytesStreamOutput(); + RemoteIndexRoutingTable indexRouting = new RemoteIndexRoutingTable(indexShardRoutingTables); + try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { indexRouting.writeTo(streamOutput); - RemoteIndexRoutingTableObject remoteIndexRoutingTable = new RemoteIndexRoutingTableObject( + RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable( streamOutput.bytes().streamInput(), metadata.index("invalid-index").getIndex() ); From e486165f703d31de3ba6252e863085859b13eba0 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 3 Jun 2024 16:50:55 +0530 Subject: [PATCH 31/50] Addressing PR comments Signed-off-by: Himshikha Gupta --- .../routing/remote/RemoteRoutingTableService.java | 12 ++++++++---- .../remote/RemoteRoutingTableServiceTests.java | 4 ---- .../remote/RemoteClusterStateServiceTests.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 5d6392be37ba6..373cd4518ad8f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; @@ -19,7 +20,6 @@ import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import java.io.Closeable; import java.io.IOException; import java.util.function.Supplier; @@ -30,7 +30,7 @@ * * @opensearch.internal */ -public class RemoteRoutingTableService implements Closeable { +public class RemoteRoutingTableService extends AbstractLifecycleComponent { /** * Cluster setting to specify if routing table should be published to remote store @@ -53,13 +53,14 @@ public RemoteRoutingTableService(Supplier repositoriesServi } @Override - public void close() throws IOException { + protected void doClose() throws IOException { if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } } - public void start() { + @Override + protected void doStart() { assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled"; final String remoteStoreRepo = settings.get( Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY @@ -70,4 +71,7 @@ public void start() { blobStoreRepository = (BlobStoreRepository) repository; } + @Override + protected void doStop() {} + } diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index d80a927914025..c3100b1eb4bd3 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -8,7 +8,6 @@ package org.opensearch.cluster.routing.remote; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.repositories.FilterRepository; @@ -30,7 +29,6 @@ public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { private RemoteRoutingTableService remoteRoutingTableService; - private ClusterSettings clusterSettings; private Supplier repositoriesServiceSupplier; private RepositoriesService repositoriesService; private BlobStoreRepository blobStoreRepository; @@ -46,8 +44,6 @@ public void setup() { .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") .build(); - clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - blobStoreRepository = mock(BlobStoreRepository.class); when(repositoriesService.repository("routing_repository")).thenReturn(blobStoreRepository); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 254d1edfcda90..6b059c8aea516 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -1402,7 +1402,7 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { "test-node-id", repositoriesServiceSupplier, newSettings, - clusterSettings, + clusterService, () -> 0L, threadPool, List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)) From b3c18d876e86da5a8423c5f3b323375771424e64 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 3 Jun 2024 18:11:16 +0530 Subject: [PATCH 32/50] test fixes Signed-off-by: Himshikha Gupta --- .../IndexRoutingTableHeaderTests.java | 5 +++-- ...sts.java => RemoteIndexRoutingTableTests.java} | 15 ++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) rename server/src/test/java/org/opensearch/gateway/remote/routingtable/{RemoteIndexRoutingTableObjectTests.java => RemoteIndexRoutingTableTests.java} (87%) diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index bea65936fc32e..a3f0ac36a40f1 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -17,13 +17,14 @@ public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { public void testIndexRoutingTableHeader() throws IOException { - IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + IndexRoutingTableHeader header = new IndexRoutingTableHeader(indexName); try (BytesStreamOutput out = new BytesStreamOutput()) { header.writeTo(out); BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in); - assertEquals("dummyIndex", headerRead.getIndexName()); + assertEquals(indexName, headerRead.getIndexName()); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java similarity index 87% rename from server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java rename to server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java index d6fed51f692a5..72066d8afb45b 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java @@ -20,21 +20,22 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; -public class RemoteIndexRoutingTableObjectTests extends OpenSearchTestCase { +public class RemoteIndexRoutingTableTests extends OpenSearchTestCase { public void testRoutingTableInput() { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); int numberOfShards = randomIntBetween(1, 10); int numberOfReplicas = randomIntBetween(1, 10); Metadata metadata = Metadata.builder() .put( - IndexMetadata.builder("test") + IndexMetadata.builder(indexName) .settings(settings(Version.CURRENT)) .numberOfShards(numberOfShards) .numberOfReplicas(numberOfReplicas) ) .build(); - RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); + RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { RemoteIndexRoutingTable indexRouting = new RemoteIndexRoutingTable(indexShardRoutingTables); @@ -42,14 +43,14 @@ public void testRoutingTableInput() { indexRouting.writeTo(streamOutput); RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable( streamOutput.bytes().streamInput(), - metadata.index("test").getIndex() + metadata.index(indexName).getIndex() ); IndexRoutingTable indexRoutingTable = remoteIndexRoutingTable.getIndexRoutingTable(); assertEquals(numberOfShards, indexRoutingTable.getShards().size()); - assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); + assertEquals(metadata.index(indexName).getIndex(), indexRoutingTable.getIndex()); assertEquals( - indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), - numberOfShards * (1 + numberOfReplicas) + numberOfShards * (1 + numberOfReplicas), + indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size() ); } catch (IOException e) { throw new RuntimeException(e); From 50a2a2ebd38bc70f6ffec0c12c20e1a263a1a235 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 3 Jun 2024 22:12:57 +0530 Subject: [PATCH 33/50] Cleaning up code Signed-off-by: Himshikha Gupta --- .../routing/IndexShardRoutingTable.java | 3 +- .../remote/RemoteRoutingTableService.java | 130 ++++++++---------- .../remote/ClusterMetadataManifest.java | 2 +- .../remote/RemoteClusterStateService.java | 15 +- .../RemoteRoutingTableServiceTests.java | 39 ++++++ 5 files changed, 116 insertions(+), 73 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index fd8cbea42c12f..b05aa2085812f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -43,6 +43,7 @@ import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.ResponseCollectorService; @@ -738,7 +739,7 @@ public boolean equals(Object o) { IndexShardRoutingTable that = (IndexShardRoutingTable) o; if (!shardId.equals(that.shardId)) return false; - if (!shards.equals(that.shards)) return false; + if(!new HashSet<>(shards).equals(new HashSet<>(that.shards))) return false; return true; } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 12980f7c7fe07..fbcc104b7fde0 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,15 +10,20 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.gateway.remote.RemoteClusterStateService; -import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInput; +import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -52,8 +57,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.opensearch.common.blobstore.transfer.RemoteTransferContainer.checksumOfChecksum; - import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; /** @@ -89,18 +92,25 @@ public RemoteRoutingTableService(Supplier repositoriesServi this.settings = settings; } - public List getChangedIndicesRouting(ClusterState previousClusterState, - ClusterState clusterState) { - Map previousIndexRoutingTable = previousClusterState.getRoutingTable().getIndicesRouting(); - List changedIndicesRouting = new ArrayList<>(); - for (IndexRoutingTable indexRouting : clusterState.getRoutingTable().getIndicesRouting().values()) { - if (!(previousIndexRoutingTable.containsKey(indexRouting.getIndex().getName()) && indexRouting.equals(previousIndexRoutingTable.get(indexRouting.getIndex().getName())))) { - changedIndicesRouting.add(indexRouting); - logger.info("changedIndicesRouting {}", indexRouting.prettyPrint()); - } + private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = new DiffableUtils.NonDiffableValueSerializer() { + @Override + public void write(IndexRoutingTable value, StreamOutput out) throws IOException { + value.writeTo(out); } - return changedIndicesRouting; + @Override + public IndexRoutingTable read(StreamInput in, String key) throws IOException { + return IndexRoutingTable.readFrom(in); + } + }; + + public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff(RoutingTable before, RoutingTable after) { + return DiffableUtils.diff( + before.getIndicesRouting(), + after.getIndicesRouting(), + DiffableUtils.getStringKeySerializer(), + CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER + ); } public CheckedRunnable getIndexRoutingAsyncAction( @@ -111,18 +121,14 @@ public CheckedRunnable getIndexRoutingAsyncAction( ) throws IOException { BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN); - logger.info("custerMetadataBasePath {}", indexRoutingPath); - BlobPath path = RemoteStoreEnums.PathType.HASHED_PREFIX.path(RemoteStorePathStrategy.BasePathInput.builder() .basePath(indexRoutingPath) .indexUUID(indexRouting.getIndex().getUUID()) .build(), RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64); - logger.info("path from prefix hasd {}", path); final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); final String fileName = getIndexRoutingFileName(); - logger.info("fileName {}", fileName); ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse( @@ -133,36 +139,48 @@ public CheckedRunnable getIndexRoutingAsyncAction( INDEX_ROUTING_METADATA_PREFIX ) ), - ex -> latchedActionListener.onFailure(new RemoteClusterStateService.RemoteStateTransferException(indexRouting.getIndex().toString(), ex)) + ex -> latchedActionListener.onFailure(new RemoteClusterStateService.RemoteStateTransferException("Exception in writing index to remote store: " + indexRouting.getIndex().toString(), ex)) ); - if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { - logger.info("TRYING FILE UPLOAD"); + return () -> uploadIndex(indexRouting, fileName , blobContainer, completionListener); + } - return () -> { - logger.info("Going to upload {}", indexRouting.prettyPrint()); - uploadIndex(indexRouting, fileName , blobContainer); - logger.info("upload done {}", indexRouting.prettyPrint()); + public List getAllUploadedIndicesRouting(ClusterMetadataManifest previousManifest, List indicesRoutingToUpload, Set indicesRoutingToDelete) { + final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() + .stream() + .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); - completionListener.onResponse(null); - logger.info("response done {}", indexRouting.prettyPrint()); + indicesRoutingToUpload.forEach( + uploadedIndexRouting -> allUploadedIndicesRouting.put(uploadedIndexRouting.getIndexName(), uploadedIndexRouting) + ); + indicesRoutingToDelete.forEach(allUploadedIndicesRouting::remove); + + return new ArrayList<>(allUploadedIndicesRouting.values()); + } - }; + private void uploadIndex(IndexRoutingTable indexRouting, String fileName, BlobContainer blobContainer, ActionListener completionListener) throws IOException { + RemoteIndexRoutingTable indexRoutingInput = new RemoteIndexRoutingTable(indexRouting); + BytesReference bytesInput = null; + try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { + indexRoutingInput.writeTo(streamOutput); + bytesInput = streamOutput.bytes(); + } catch (IOException e) { + throw new IOException("Failed to serialize IndexRoutingTable. ", e); } - logger.info("TRYING S3 UPLOAD"); + if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { + try { + blobContainer.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); + completionListener.onResponse(null); + } catch (IOException e) { + throw new IOException("Failed to write IndexRoutingTable to remote store. ", e); + } + return; + } - IndexRoutingTableInput indexRoutingInput = new IndexRoutingTableInput(indexRouting); - BytesReference bytesInput = indexRoutingInput.write(); try ( IndexInput input = new ByteArrayIndexInput("indexrouting",BytesReference.toBytes(bytesInput))) { - long expectedChecksum; - try { - expectedChecksum = checksumOfChecksum(input.clone(), 8); - } catch (Exception e) { - throw e; - } try ( RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( fileName, @@ -171,46 +189,18 @@ public CheckedRunnable getIndexRoutingAsyncAction( true, WritePriority.URGENT, (size, position) -> new OffsetRangeIndexInputStream(input, size, position), - expectedChecksum, - ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported() + null, + false ) ) { - return () -> ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), completionListener); + ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), completionListener); } catch (IOException e) { - e.printStackTrace(); - return null; + throw new IOException("Failed to write IndexRoutingTable to remote store. ", e); } - } - } - - - public List getAllUploadedIndicesRouting(ClusterMetadataManifest previousManifest, List indicesRoutingToUpload, Set indicesRoutingToDelete) { - final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() - .stream() - .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); - - indicesRoutingToUpload.forEach( - uploadedIndexRouting -> allUploadedIndicesRouting.put(uploadedIndexRouting.getIndexName(), uploadedIndexRouting) - ); - - indicesRoutingToDelete.forEach(index -> allUploadedIndicesRouting.remove(index)); - - logger.info("allUploadedIndicesRouting ROUTING {}", allUploadedIndicesRouting); - - return new ArrayList<>(allUploadedIndicesRouting.values()); - } - - private void uploadIndex(IndexRoutingTable indexRouting, String fileName, BlobContainer container) { - logger.info("Starting write"); - - try { - IndexRoutingTableInput indexRoutingInput = new IndexRoutingTableInput(indexRouting); - BytesReference bytesInput = indexRoutingInput.write(); - container.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); - logger.info("SUccessful write"); } catch (IOException e) { - logger.error("Failed to write {}", e); + throw new IOException("Failed to create transfer object for IndexRoutingTable for remote store upload. ", e); } + } private String getIndexRoutingFileName() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 9179d631d3409..a46981f8e0785 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -41,7 +41,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. - public static final int CODEC_V3 = 3; // In Codec V2 we introduce index routing-metadata in manifest file. + public static final int CODEC_V3 = 3; // In Codec V3 we introduce index routing-metadata in manifest file. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 58ad3547d774e..7fb2bef4acb6f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -14,6 +14,7 @@ import org.opensearch.Version; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; @@ -168,6 +169,15 @@ public class RemoteClusterStateService implements Closeable { /** * Manifest format compatible with codec v2, where global metadata file is replaced with multiple metadata attribute files */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V2 = new ChecksumBlobStoreFormat<>( + "cluster-metadata-manifest", + METADATA_MANIFEST_NAME_FORMAT, + ClusterMetadataManifest::fromXContentV2 + ); + + /** + * Manifest format compatible with codec v3, where global metadata file is replaced with multiple metadata attribute files + */ public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( "cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, @@ -379,7 +389,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( List indicesRoutingToUpload = new ArrayList<>(); if(remoteRoutingTableService.isPresent()) { - indicesRoutingToUpload = remoteRoutingTableService.get().getChangedIndicesRouting(previousClusterState, clusterState); + DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService.getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable()); + routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v)); } UploadedMetadataResults uploadedMetadataResults; @@ -1506,6 +1517,8 @@ private ChecksumBlobStoreFormat getClusterMetadataManif long codecVersion = getManifestCodecVersion(fileName); if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V2) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V2; } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { return CLUSTER_METADATA_MANIFEST_FORMAT_V1; } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index c3100b1eb4bd3..63718b1edbae8 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -8,8 +8,22 @@ package org.opensearch.cluster.routing.remote; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.Index; +import org.opensearch.gateway.remote.RemoteClusterStateServiceTests; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; @@ -18,6 +32,7 @@ import org.junit.After; import org.junit.Before; +import java.util.Map; import java.util.function.Supplier; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; @@ -75,4 +90,28 @@ public void testFailStartWhenNotBlobRepository() { assertThrows(AssertionError.class, () -> remoteRoutingTableService.start()); } + public void testGetChangedIndicesRouting() { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + final Index index = new Index(indexName, "uuid"); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(1).numberOfReplicas(1).build(); + + RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).routingTable(routingTable).build(); + + assertEquals(0, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), state.getRoutingTable()).getUpserts().size()); + + //Reversing order to check for equality without order. + IndexRoutingTable indexRouting = routingTable.getIndicesRouting().get(indexName); + IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(index).addShard(indexRouting.getShards().get(0).replicaShards().get(0)) + .addShard(indexRouting.getShards().get(0).primaryShard()).build(); + ClusterState newState = ClusterState.builder(ClusterName.DEFAULT).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build(); + + assertEquals(0, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), newState.getRoutingTable()).getUpserts().size()); + } + } From 8ecf8652cceec33c7cde3ef38c4e63803b800dac Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 4 Jun 2024 16:17:55 +0530 Subject: [PATCH 34/50] addressing PR comments Signed-off-by: Himshikha Gupta --- .../gateway/remote/ClusterMetadataManifest.java | 2 +- .../gateway/remote/RemoteClusterStateService.java | 1 + .../remote/routingtable/RemoteIndexRoutingTable.java | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 72e3826a3223a..b3b1bf37f8696 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -41,7 +41,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. - public static final int CODEC_V3 = 3; // In Codec V2 we introduce index routing-metadata in manifest file. + public static final int CODEC_V3 = 3; // In Codec V3, we introduce index routing-metadata in manifest file. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 9e9caf47a0997..9a0db3a2d25fd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -797,6 +797,7 @@ private ClusterMetadataManifest uploadManifest( uploadedTemplatesMetadata, uploadedCustomMetadataMap, clusterState.routingTable().version(), + // TODO: Add actual list of changed indices routing with index routing upload flow. new ArrayList<>() ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java index 5641ef01df8d2..c584e8400939f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java @@ -32,6 +32,12 @@ public RemoteIndexRoutingTable(IndexRoutingTable indexRoutingTable) { this.indexRoutingTable = indexRoutingTable; } + /** + * Reads data from inputStream and creates RemoteIndexRoutingTable object with the {@link IndexRoutingTable} + * @param inputStream + * @param index + * @throws IOException + */ public RemoteIndexRoutingTable(InputStream inputStream, Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) { @@ -57,6 +63,11 @@ public IndexRoutingTable getIndexRoutingTable() { return indexRoutingTable; } + /** + * Writes {@link IndexRoutingTable} to the given stream + * @param streamOutput + * @throws IOException + */ @Override public void writeTo(StreamOutput streamOutput) throws IOException { try { From dfc904207e132b725f190c99da8b2db2b503262f Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 4 Jun 2024 15:59:16 +0530 Subject: [PATCH 35/50] node join based on routing table repo Signed-off-by: Himshikha Gupta --- .../coordination/JoinTaskExecutor.java | 41 +++++++++++----- .../metadata/RepositoriesMetadata.java | 9 +++- .../remote/RemoteRoutingTableService.java | 10 ---- .../common/settings/ClusterSettings.java | 6 +-- .../main/java/org/opensearch/node/Node.java | 2 +- .../remotestore/RemoteStoreNodeAttribute.java | 32 ++++--------- .../remotestore/RemoteStoreNodeService.java | 14 ++---- .../coordination/JoinTaskExecutorTests.java | 41 +++------------- .../RemoteRoutingTableServiceTests.java | 1 - .../RemoteClusterStateServiceTests.java | 2 - .../node/RemoteStoreNodeAttributeTests.java | 47 +++++-------------- 11 files changed, 69 insertions(+), 136 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index cb428eb9c3e45..fb8ab0c250b81 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -187,8 +187,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo DiscoveryNode dn = remoteDN.orElseGet(() -> (currentNodes.getNodes().values()).stream().findFirst().get()); RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( dn, - currentState.getMetadata().custom(RepositoriesMetadata.TYPE), - currentState.getMetadata().settings() + currentState.getMetadata().custom(RepositoriesMetadata.TYPE) ); assert nodesBuilder.isLocalNodeElectedClusterManager(); @@ -224,8 +223,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo logger.info("Updating system repository now for remote store"); repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( node, - currentState.getMetadata().custom(RepositoriesMetadata.TYPE), - currentState.getMetadata().settings() + currentState.getMetadata().custom(RepositoriesMetadata.TYPE) ); } @@ -513,11 +511,28 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod assert existingNodes.isEmpty() == false; CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); + + List reposToSkip = new ArrayList<>(1); + Optional remoteRoutingTableNode = existingNodes.stream() + .filter( + node -> node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null + ) + .findFirst(); + // If none of the existing nodes have routing table repo, then we skip this repo check if present in joining node. + // This ensures a new node with remote routing table repo is able to join the cluster. + if (remoteRoutingTableNode.isEmpty()) { + String joiningNodeRepoName = joiningNode.getAttributes() + .get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); + if (joiningNodeRepoName != null) { + reposToSkip.add(joiningNodeRepoName); + } + } + if (STRICT.equals(remoteStoreCompatibilityMode)) { DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { - ensureRemoteStoreNodesCompatibility(joiningNode, existingNode, metadata.settings()); + ensureRemoteStoreNodesCompatibility(joiningNode, existingNode, reposToSkip); } else { if (existingNode.isRemoteStoreNode()) { throw new IllegalStateException( @@ -540,20 +555,22 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod } if (joiningNode.isRemoteStoreNode()) { Optional remoteDN = existingNodes.stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst(); - remoteDN.ifPresent( - discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode, metadata.settings()) - ); + remoteDN.ifPresent(discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode, reposToSkip)); } } } } - private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNode existingNode, Settings settings) { + private static void ensureRemoteStoreNodesCompatibility( + DiscoveryNode joiningNode, + DiscoveryNode existingNode, + List reposToSkip + ) { if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { - RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode, settings); - RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode, settings); - if (existingRemoteStoreNodeAttribute.equalsIgnoreOptionalRepo(joiningRemoteStoreNodeAttribute) == false) { + RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); + RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode); + if (existingRemoteStoreNodeAttribute.equalsWithRepoSkip(joiningRemoteStoreNodeAttribute, reposToSkip) == false) { throw new IllegalStateException( "a remote store node [" + joiningNode diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index 2c9eac78112d2..4b3dc7964a87b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -51,6 +51,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.EnumSet; import java.util.List; import java.util.stream.Collectors; @@ -172,7 +173,7 @@ public boolean equalsIgnoreGenerations(@Nullable RepositoriesMetadata other) { * @param reposToSkip list of repos to skip check for equality * @return {@code true} iff both instances contain the same repositories apart from differences in generations, not including repos provided in reposToSkip. */ - public boolean equalsIgnoreGenerationsIgnoreOptionalRepos(@Nullable RepositoriesMetadata other, List reposToSkip) { + public boolean equalsIgnoreGenerationsWithRepoSkip(@Nullable RepositoriesMetadata other, List reposToSkip) { if (other == null) { return false; } @@ -182,9 +183,15 @@ public boolean equalsIgnoreGenerationsIgnoreOptionalRepos(@Nullable Repositories List otherRepositories = other.repositories.stream() .filter(repo -> !reposToSkip.contains(repo.name())) .collect(Collectors.toList()); + if (otherRepositories.size() != currentRepositories.size()) { return false; } + // Sort repos by name for ordered comparison + Comparator compareByName = (o1, o2) -> o1.name().compareTo(o2.name()); + currentRepositories.sort(compareByName); + otherRepositories.sort(compareByName); + for (int i = 0; i < currentRepositories.size(); i++) { if (currentRepositories.get(i).equalsIgnoreGenerations(otherRepositories.get(i)) == false) { return false; diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 373cd4518ad8f..ba2208e17df1f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; -import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.node.Node; @@ -32,15 +31,6 @@ */ public class RemoteRoutingTableService extends AbstractLifecycleComponent { - /** - * Cluster setting to specify if routing table should be published to remote store - */ - public static final Setting REMOTE_ROUTING_TABLE_ENABLED_SETTING = Setting.boolSetting( - "cluster.remote_store.routing_table.enabled", - false, - Setting.Property.NodeScope, - Setting.Property.Final - ); private static final Logger logger = LogManager.getLogger(RemoteRoutingTableService.class); private final Settings settings; private final Supplier repositoriesService; diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 3d45cf5c872dc..297fc98764d07 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -77,7 +77,6 @@ import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterManagerService; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; @@ -744,10 +743,7 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS, - RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, - - // Remote Routing table settings - RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING + RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA ) ) ); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index dc4f29d07abce..49545fa8a0c8b 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1983,7 +1983,7 @@ public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { ); if (isRemoteStoreAttributePresent(settings)) { - remoteStoreNodeService.createAndVerifyRepositories(discoveryNode, settings); + remoteStoreNodeService.createAndVerifyRepositories(discoveryNode); } localNode.set(discoveryNode); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index e35229f02a554..a0f745a4270c4 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -12,7 +12,6 @@ import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.gateway.remote.RemoteClusterStateService; @@ -53,7 +52,6 @@ public class RemoteStoreNodeAttribute { public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing_table.repository"; private final RepositoriesMetadata repositoriesMetadata; - private final List optionalRepos; public static List SUPPORTED_DATA_REPO_NAME_ATTRIBUTES = List.of( REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, @@ -63,18 +61,8 @@ public class RemoteStoreNodeAttribute { /** * Creates a new {@link RemoteStoreNodeAttribute} */ - public RemoteStoreNodeAttribute(DiscoveryNode node, Settings settings) { + public RemoteStoreNodeAttribute(DiscoveryNode node) { this.repositoriesMetadata = buildRepositoriesMetadata(node); - // For supporting feature launches where new repos are added, we can mark repos to be optional and ensure node joins are not - // impacted due to diff in repos. - this.optionalRepos = new ArrayList<>(); - if (!RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings)) { - if (node.getAttributes().containsKey(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - optionalRepos.add( - node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) - ); - } - } } private String validateAttributeNonNull(DiscoveryNode node, String attributeKey) { @@ -214,9 +202,7 @@ private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { } public static boolean isRemoteRoutingTableEnabled(Settings settings) { - return FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) - && RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.get(settings) - && isRemoteRoutingTableAttributePresent(settings); + return FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) && isRemoteRoutingTableAttributePresent(settings); } public RepositoriesMetadata getRepositoriesMetadata() { @@ -264,20 +250,18 @@ public int hashCode() { } /** - * Checks if 2 instances are equal, ignoring optionalRepos for both. - * + * Checks if 2 instances are equal, with option to skip check for a list of repos. + * * * @param o other instance - * @return {@code true} iff both instances are equal, not including the repositories in both instances if they are part of optionalRepos. + * @param reposToSkip list of repos to skip check for equality + * @return {@code true} iff both instances are equal, not including the repositories in both instances if they are part of reposToSkip. */ - public boolean equalsIgnoreOptionalRepo(Object o) { + public boolean equalsWithRepoSkip(Object o, List reposToSkip) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RemoteStoreNodeAttribute that = (RemoteStoreNodeAttribute) o; - List reposToSkip = new ArrayList<>(); - reposToSkip.addAll(this.optionalRepos); - reposToSkip.addAll(that.optionalRepos); - return this.getRepositoriesMetadata().equalsIgnoreGenerationsIgnoreOptionalRepos(that.getRepositoriesMetadata(), reposToSkip); + return this.getRepositoriesMetadata().equalsIgnoreGenerationsWithRepoSkip(that.getRepositoriesMetadata(), reposToSkip); } @Override diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 42d8fcde5cbff..874c9408de6c5 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -17,7 +17,6 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; -import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -145,8 +144,8 @@ public RemoteStoreNodeService(Supplier repositoriesService, * If the creation or verification fails this will close all the repositories this method created and throw * exception. */ - public void createAndVerifyRepositories(DiscoveryNode localNode, Settings settings) { - RemoteStoreNodeAttribute nodeAttribute = new RemoteStoreNodeAttribute(localNode, settings); + public void createAndVerifyRepositories(DiscoveryNode localNode) { + RemoteStoreNodeAttribute nodeAttribute = new RemoteStoreNodeAttribute(localNode); RepositoriesService reposService = repositoriesService.get(); Map repositories = new HashMap<>(); for (RepositoryMetadata repositoryMetadata : nodeAttribute.getRepositoriesMetadata().repositories()) { @@ -178,15 +177,10 @@ public void createAndVerifyRepositories(DiscoveryNode localNode, Settings settin * repository is already present in the cluster state and if it's different then the joining remote store backed * node repository metadata an exception will be thrown and the node will not be allowed to join the cluster. */ - public RepositoriesMetadata updateRepositoriesMetadata( - DiscoveryNode joiningNode, - RepositoriesMetadata existingRepositories, - Settings settings - ) { + public RepositoriesMetadata updateRepositoriesMetadata(DiscoveryNode joiningNode, RepositoriesMetadata existingRepositories) { if (joiningNode.isRemoteStoreNode()) { List updatedRepositoryMetadataList = new ArrayList<>(); - List newRepositoryMetadataList = new RemoteStoreNodeAttribute(joiningNode, settings) - .getRepositoriesMetadata() + List newRepositoryMetadataList = new RemoteStoreNodeAttribute(joiningNode).getRepositoriesMetadata() .repositories(); if (existingRepositories == null) { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 6980e9e670a19..79751b863c6e4 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -49,13 +49,10 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.allocation.AllocationService; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.node.Node; -import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -205,9 +202,7 @@ public void testUpdatesNodeWithNewRoles() throws Exception { when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); final RemoteStoreNodeService remoteStoreNodeService = mock(RemoteStoreNodeService.class); - when(remoteStoreNodeService.updateRepositoriesMetadata(any(), any(), any())).thenReturn( - new RepositoriesMetadata(Collections.emptyList()) - ); + when(remoteStoreNodeService.updateRepositoriesMetadata(any(), any())).thenReturn(new RepositoriesMetadata(Collections.emptyList())); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -950,7 +945,7 @@ public void testNodeJoinInMixedMode() { JoinTaskExecutor.ensureNodesCompatibility(joiningNode2, currentNodes, metadata); } - public void testRemoteRoutingTableDisabledNodeJoin() { + public void testRemoteRoutingTableRepoAbsentNodeJoin() { final DiscoveryNode existingNode = new DiscoveryNode( UUIDs.base64UUID(), @@ -968,7 +963,7 @@ public void testRemoteRoutingTableDisabledNodeJoin() { JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } - public void testRemoteRoutingTableDisabledNodeJoinRepoPresentInJoiningNode() { + public void testRemoteRoutingTableNodeJoinRepoPresentInJoiningNode() { final DiscoveryNode existingNode = new DiscoveryNode( UUIDs.base64UUID(), buildNewFakeTransportAddress(), @@ -986,7 +981,7 @@ public void testRemoteRoutingTableDisabledNodeJoinRepoPresentInJoiningNode() { JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } - public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInExistingNode() { + public void testRemoteRoutingTableNodeJoinRepoPresentInExistingNode() { Map attr = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); attr.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); final DiscoveryNode existingNode = new DiscoveryNode( @@ -996,20 +991,9 @@ public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInExistingNode() { DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT ); - final Settings settings = Settings.builder() - .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true") - .put( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, - ROUTING_TABLE_REPO - ) - .put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true") - .build(); - final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); - Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) - .metadata(metadata) .build(); DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); @@ -1019,7 +1003,7 @@ public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInExistingNode() { ); } - public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInBothNode() { + public void testRemoteRoutingTableNodeJoinRepoPresentInBothNode() { Map attr = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); attr.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); final DiscoveryNode existingNode = new DiscoveryNode( @@ -1029,20 +1013,9 @@ public void testRemoteRoutingTableEnabledNodeJoinRepoPresentInBothNode() { DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT ); - final Settings settings = Settings.builder() - .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true") - .put( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, - ROUTING_TABLE_REPO - ) - .put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true") - .build(); - final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); - Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) - .metadata(metadata) .build(); DiscoveryNode joiningNode = newDiscoveryNode(attr); diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index c3100b1eb4bd3..9a9cbfa153259 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -40,7 +40,6 @@ public void setup() { when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); Settings settings = Settings.builder() - .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), true) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") .build(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 6b059c8aea516..324023061ac46 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -19,7 +19,6 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; @@ -1388,7 +1387,6 @@ public void testRemoteRoutingTableNotInitializedWhenDisabled() { public void testRemoteRoutingTableInitializedWhenEnabled() { Settings newSettings = Settings.builder() - .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), true) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index eb293287a36c3..de7f8977686a7 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -12,15 +12,14 @@ import org.opensearch.cluster.metadata.CryptoMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.test.OpenSearchTestCase; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -75,7 +74,7 @@ public void testCryptoMetadata() throws UnknownHostException { Version.CURRENT ); - RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node, Settings.builder().build()); + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); assertEquals(remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().size(), 1); RepositoryMetadata repositoryMetadata = remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().get(0); Settings.Builder settings = Settings.builder(); @@ -117,7 +116,7 @@ public void testInvalidCryptoMetadata() throws UnknownHostException { Version.CURRENT ); - assertThrows(IllegalStateException.class, () -> new RemoteStoreNodeAttribute(node, Settings.builder().build())); + assertThrows(IllegalStateException.class, () -> new RemoteStoreNodeAttribute(node)); } public void testNoCryptoMetadata() throws UnknownHostException { @@ -146,14 +145,13 @@ public void testNoCryptoMetadata() throws UnknownHostException { Version.CURRENT ); - RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node, Settings.builder().build()); + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); assertEquals(remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().size(), 1); RepositoryMetadata repositoryMetadata = remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().get(0); assertNull(repositoryMetadata.cryptoMetadata()); } - public void testEqualsIgnoreOptionalRepo() throws UnknownHostException { - // Node 1 -> Remote Routing disabled, repo not present, Node 2 -> Remote Routing disabled, repo present -> should succeed + public void testEqualsWithRepoSkip() throws UnknownHostException { String repoName = "remote-store-A"; String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); String repoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repoName); @@ -171,7 +169,7 @@ public void testEqualsIgnoreOptionalRepo() throws UnknownHostException { repoSettingsKey + "base_path", "xyz" ); - DiscoveryNode nodeWithoutRoutingTableAttr = new DiscoveryNode( + DiscoveryNode node = new DiscoveryNode( "C", new TransportAddress(InetAddress.getByName("localhost"), 9876), attr, @@ -179,10 +177,7 @@ public void testEqualsIgnoreOptionalRepo() throws UnknownHostException { Version.CURRENT ); - RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute( - nodeWithoutRoutingTableAttr, - Settings.builder().build() - ); + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); String routingTableRepoName = "remote-store-B"; String routingTableRepoTypeSettingKey = String.format( @@ -216,36 +211,16 @@ public void testEqualsIgnoreOptionalRepo() throws UnknownHostException { routingTableRepoSettingsKey, "xyz" ); - DiscoveryNode nodeWithRoutingTableAttr = new DiscoveryNode( + DiscoveryNode node2 = new DiscoveryNode( "C", new TransportAddress(InetAddress.getByName("localhost"), 9876), attr2, emptySet(), Version.CURRENT ); - RemoteStoreNodeAttribute remoteStoreNodeAttribute2 = new RemoteStoreNodeAttribute( - nodeWithRoutingTableAttr, - Settings.builder().build() - ); - assertTrue(remoteStoreNodeAttribute.equalsIgnoreOptionalRepo(remoteStoreNodeAttribute2)); - - // Node 1 -> Remote Routing enabled, repo present, Node 2 -> Remote Routing enabled, repo present -> should succeed - final Settings nodeSettings = Settings.builder().put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); - RemoteStoreNodeAttribute remoteStoreNodeAttribute3 = new RemoteStoreNodeAttribute( - nodeWithRoutingTableAttr, - Settings.builder() - .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true") - .put(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName) - .build() - ); - assertTrue(remoteStoreNodeAttribute3.equalsIgnoreOptionalRepo(remoteStoreNodeAttribute2)); + RemoteStoreNodeAttribute remoteStoreNodeAttribute2 = new RemoteStoreNodeAttribute(node2); - // Node 1 -> Remote Routing enabled, repo present, Node 2 -> Remote Routing enabled, repo not present -> should fail - RemoteStoreNodeAttribute remoteStoreNodeAttribute4 = new RemoteStoreNodeAttribute( - nodeWithoutRoutingTableAttr, - Settings.builder().put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), "true").build() - ); - assertFalse(remoteStoreNodeAttribute3.equalsIgnoreOptionalRepo(remoteStoreNodeAttribute4)); + assertFalse(remoteStoreNodeAttribute.equalsWithRepoSkip(remoteStoreNodeAttribute2, List.of())); + assertTrue(remoteStoreNodeAttribute.equalsWithRepoSkip(remoteStoreNodeAttribute2, List.of(routingTableRepoName))); } } From 7ad7d05043cb0fce893592d8a5c0a9288b18ab39 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 5 Jun 2024 12:19:43 +0530 Subject: [PATCH 36/50] rebasing Signed-off-by: Himshikha Gupta --- CHANGELOG.md | 1 + .../org/opensearch/gateway/remote/RemoteClusterStateService.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ce11dabb6e9c..1ffb438172f50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819)) - [Remote State] Add async remote state deletion task running on an interval, configurable by a setting ([#13131](https://github.com/opensearch-project/OpenSearch/pull/13131)) - Allow setting query parameters on requests ([#13776](https://github.com/opensearch-project/OpenSearch/issues/13776)) +- Add remote routing table for remote state publication with experimental feature flag ([#13304](https://github.com/opensearch-project/OpenSearch/pull/13304)) ### Dependencies - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559)) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index f8d09c5726d99..1a67f3cf25bbf 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -19,7 +19,6 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; From 0e5862dc27fccd96b826fe5fa6c618311fcd705a Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 5 Jun 2024 13:49:01 +0530 Subject: [PATCH 37/50] handle case in node join Signed-off-by: Himshikha Gupta --- .../coordination/JoinTaskExecutor.java | 7 ++- .../coordination/JoinTaskExecutorTests.java | 62 +++++++++++++++++++ .../RemoteClusterStateServiceTests.java | 2 +- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index fb8ab0c250b81..f77a7ffc8ce8e 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -529,8 +529,7 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod } if (STRICT.equals(remoteStoreCompatibilityMode)) { - - DiscoveryNode existingNode = existingNodes.get(0); + DiscoveryNode existingNode = remoteRoutingTableNode.orElseGet(() -> existingNodes.get(0)); if (joiningNode.isRemoteStoreNode()) { ensureRemoteStoreNodesCompatibility(joiningNode, existingNode, reposToSkip); } else { @@ -554,7 +553,9 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod throw new IllegalStateException(reason); } if (joiningNode.isRemoteStoreNode()) { - Optional remoteDN = existingNodes.stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst(); + Optional remoteDN = remoteRoutingTableNode.isPresent() + ? remoteRoutingTableNode + : existingNodes.stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst(); remoteDN.ifPresent(discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode, reposToSkip)); } } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 79751b863c6e4..9cb1bd0b57132 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -1022,6 +1022,68 @@ public void testRemoteRoutingTableNodeJoinRepoPresentInBothNode() { JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } + public void testRemoteRoutingTableNodeJoinNodeWithRemoteAndRoutingRepoDifference() { + Map attr = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + attr.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + attr, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final DiscoveryNode existingNode2 = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode2).add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(attr); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + + public void testRemoteRoutingTableNodeJoinNodeWithRemoteAndRoutingRepoDifferenceMixedMode() { + Map attr = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + attr.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + attr, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final DiscoveryNode existingNode2 = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final Settings settings = Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE) + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + .build(); + final Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode2).add(existingNode).localNodeId(existingNode.getId()).build()) + .metadata(metadata) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(attr); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode existingNode, int expectedRepositories) throws Exception { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index e9f717dd3cf3b..aa5996d734d27 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -1518,7 +1518,7 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { "test-node-id", repositoriesServiceSupplier, newSettings, - clusterService, + clusterSettings, () -> 0L, threadPool, List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)) From bc1c1c772e8636524a2e3fb71a6d6bd279bb1c8f Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 5 Jun 2024 17:28:49 +0530 Subject: [PATCH 38/50] Adding tests Signed-off-by: Himshikha Gupta --- .../routing/IndexShardRoutingTable.java | 3 +- .../remote/RemoteRoutingTableService.java | 121 ++++++++++-------- .../remote/ClusterMetadataManifest.java | 4 +- .../remote/RemoteClusterStateService.java | 43 ++++--- .../routingtable/RemoteIndexRoutingTable.java | 2 - .../index/remote/RemoteStoreEnums.java | 39 +++--- .../index/remote/RemoteStorePathStrategy.java | 4 +- .../RemoteRoutingTableServiceTests.java | 30 +++-- 8 files changed, 131 insertions(+), 115 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index b05aa2085812f..39589f46f7a4e 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -43,7 +43,6 @@ import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.node.ResponseCollectorService; @@ -739,7 +738,7 @@ public boolean equals(Object o) { IndexShardRoutingTable that = (IndexShardRoutingTable) o; if (!shardId.equals(that.shardId)) return false; - if(!new HashSet<>(shards).equals(new HashSet<>(that.shards))) return false; + if (!new HashSet<>(shards).equals(new HashSet<>(that.shards))) return false; return true; } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 00273d74f1c34..b41d8456e40bc 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,19 +10,34 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.IndexInput; +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; +import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.index.remote.RemoteStorePathStrategy; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -30,30 +45,12 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.io.IOException; -import java.util.function.Supplier; - -import org.apache.lucene.store.IndexInput; -import org.opensearch.action.LatchedActionListener; -import org.opensearch.cluster.ClusterState; -import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; -import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.blobstore.stream.write.WritePriority; -import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; - -import org.opensearch.common.lucene.store.ByteArrayIndexInput; -import org.opensearch.core.action.ActionListener; -import org.opensearch.gateway.remote.ClusterMetadataManifest; -import org.opensearch.index.remote.RemoteStoreEnums; -import org.opensearch.index.remote.RemoteStorePathStrategy; -import org.opensearch.index.remote.RemoteStoreUtils; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; @@ -81,19 +78,23 @@ public RemoteRoutingTableService(Supplier repositoriesServi this.settings = settings; } - private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = new DiffableUtils.NonDiffableValueSerializer() { - @Override - public void write(IndexRoutingTable value, StreamOutput out) throws IOException { - value.writeTo(out); - } + private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = + new DiffableUtils.NonDiffableValueSerializer() { + @Override + public void write(IndexRoutingTable value, StreamOutput out) throws IOException { + value.writeTo(out); + } - @Override - public IndexRoutingTable read(StreamInput in, String key) throws IOException { - return IndexRoutingTable.readFrom(in); - } - }; + @Override + public IndexRoutingTable read(StreamInput in, String key) throws IOException { + return IndexRoutingTable.readFrom(in); + } + }; - public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff(RoutingTable before, RoutingTable after) { + public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff( + RoutingTable before, + RoutingTable after + ) { return DiffableUtils.diff( before.getIndicesRouting(), after.getIndicesRouting(), @@ -110,11 +111,10 @@ public CheckedRunnable getIndexRoutingAsyncAction( ) throws IOException { BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN); - BlobPath path = RemoteStoreEnums.PathType.HASHED_PREFIX.path(RemoteStorePathStrategy.BasePathInput.builder() - .basePath(indexRoutingPath) - .indexUUID(indexRouting.getIndex().getUUID()) - .build(), - RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64); + BlobPath path = RemoteStoreEnums.PathType.HASHED_PREFIX.path( + RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(), + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64 + ); final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); final String fileName = getIndexRoutingFileName(); @@ -128,14 +128,22 @@ public CheckedRunnable getIndexRoutingAsyncAction( INDEX_ROUTING_METADATA_PREFIX ) ), - ex -> latchedActionListener.onFailure(new RemoteClusterStateService.RemoteStateTransferException("Exception in writing index to remote store: " + indexRouting.getIndex().toString(), ex)) + ex -> latchedActionListener.onFailure( + new RemoteClusterStateService.RemoteStateTransferException( + "Exception in writing index to remote store: " + indexRouting.getIndex().toString(), + ex + ) + ) ); - return () -> uploadIndex(indexRouting, fileName , blobContainer, completionListener); + return () -> uploadIndex(indexRouting, fileName, blobContainer, completionListener); } - - public List getAllUploadedIndicesRouting(ClusterMetadataManifest previousManifest, List indicesRoutingToUpload, Set indicesRoutingToDelete) { + public List getAllUploadedIndicesRouting( + ClusterMetadataManifest previousManifest, + List indicesRoutingToUpload, + Set indicesRoutingToDelete + ) { final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() .stream() .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); @@ -148,7 +156,12 @@ public List getAllUploadedIndices return new ArrayList<>(allUploadedIndicesRouting.values()); } - private void uploadIndex(IndexRoutingTable indexRouting, String fileName, BlobContainer blobContainer, ActionListener completionListener) throws IOException { + private void uploadIndex( + IndexRoutingTable indexRouting, + String fileName, + BlobContainer blobContainer, + ActionListener completionListener + ) throws IOException { RemoteIndexRoutingTable indexRoutingInput = new RemoteIndexRoutingTable(indexRouting); BytesReference bytesInput = null; try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { @@ -159,17 +172,16 @@ private void uploadIndex(IndexRoutingTable indexRouting, String fileName, BlobCo } if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { - try { - blobContainer.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); - completionListener.onResponse(null); - } catch (IOException e) { - throw new IOException("Failed to write IndexRoutingTable to remote store. ", e); - } - return; + try { + blobContainer.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); + completionListener.onResponse(null); + } catch (IOException e) { + throw new IOException("Failed to write IndexRoutingTable to remote store. ", e); + } + return; } - try ( - IndexInput input = new ByteArrayIndexInput("indexrouting",BytesReference.toBytes(bytesInput))) { + try (IndexInput input = new ByteArrayIndexInput("indexrouting", BytesReference.toBytes(bytesInput))) { try ( RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( fileName, @@ -182,7 +194,10 @@ private void uploadIndex(IndexRoutingTable indexRouting, String fileName, BlobCo false ) ) { - ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), completionListener); + ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload( + remoteTransferContainer.createWriteContext(), + completionListener + ); } catch (IOException e) { throw new IOException("Failed to write IndexRoutingTable to remote store. ", e); } @@ -193,11 +208,7 @@ private void uploadIndex(IndexRoutingTable indexRouting, String fileName, BlobCo } private String getIndexRoutingFileName() { - return String.join( - DELIMITER, - INDEX_ROUTING_FILE_PREFIX, - RemoteStoreUtils.invertLong(System.currentTimeMillis()) - ); + return String.join(DELIMITER, INDEX_ROUTING_FILE_PREFIX, RemoteStoreUtils.invertLong(System.currentTimeMillis())); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 1ac33b0df5109..6cad12ab01769 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -844,7 +844,7 @@ private static String componentPrefix(Object[] fields) { private final String uploadedFilename; public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName) { - this( indexName,indexUUID,uploadedFileName, COMPONENT_PREFIX); + this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX); } public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) { @@ -887,8 +887,6 @@ public String getComponentPrefix() { return componentPrefix; } - - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index ecb66228b061c..920198090756c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -170,11 +170,8 @@ public class RemoteClusterStateService implements Closeable { /** * Manifest format compatible with codec v2, where global metadata file is replaced with multiple metadata attribute files */ - public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V2 = new ChecksumBlobStoreFormat<>( - "cluster-metadata-manifest", - METADATA_MANIFEST_NAME_FORMAT, - ClusterMetadataManifest::fromXContentV2 - ); + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V2 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV2); /** * Manifest format compatible with codec v3, where global metadata file is replaced with multiple metadata attribute files @@ -312,7 +309,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedTemplatesMetadata, uploadedMetadataResults.uploadedCustomMetadataMap, uploadedMetadataResults.uploadedIndicesRoutingMetadata, - false + false ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); @@ -395,8 +392,9 @@ public ClusterMetadataManifest writeIncrementalMetadata( } List indicesRoutingToUpload = new ArrayList<>(); - if(remoteRoutingTableService.isPresent()) { - DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService.getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable()); + if (remoteRoutingTableService.isPresent()) { + DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService + .getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable()); routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v)); } @@ -433,8 +431,13 @@ public ClusterMetadataManifest writeIncrementalMetadata( indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove); List allUploadedIndicesRouting = new ArrayList<>(); - if(remoteRoutingTableService.isPresent()) { - allUploadedIndicesRouting = remoteRoutingTableService.get().getAllUploadedIndicesRouting(previousManifest, uploadedMetadataResults.uploadedIndicesRoutingMetadata, indicesToBeDeletedFromRemote.keySet()); + if (remoteRoutingTableService.isPresent()) { + allUploadedIndicesRouting = remoteRoutingTableService.get() + .getAllUploadedIndicesRouting( + previousManifest, + uploadedMetadataResults.uploadedIndicesRoutingMetadata, + indicesToBeDeletedFromRemote.keySet() + ); } final ClusterMetadataManifest manifest = uploadManifest( @@ -496,7 +499,8 @@ private UploadedMetadataResults writeMetadataInParallel( ) throws IOException { assert Objects.nonNull(indexMetadataUploadListeners) : "indexMetadataUploadListeners can not be null"; int totalUploadTasks = indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size() - + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + indicesRoutingToUpload.size(); + + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + + indicesRoutingToUpload.size(); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new HashMap<>(totalUploadTasks); Map results = new HashMap<>(totalUploadTasks); @@ -567,8 +571,13 @@ private UploadedMetadataResults writeMetadataInParallel( try { uploadTasks.put( indexRoutingTable.getIndex().getName() + "--indexRouting", - remoteRoutingTableService.get().getIndexRoutingAsyncAction(clusterState, indexRoutingTable, listener, getCusterMetadataBasePath(clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID())) + remoteRoutingTableService.get() + .getIndexRoutingAsyncAction( + clusterState, + indexRoutingTable, + listener, + getCusterMetadataBasePath(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()) + ) ); } catch (IOException e) { e.printStackTrace(); @@ -621,10 +630,10 @@ private UploadedMetadataResults writeMetadataInParallel( } UploadedMetadataResults response = new UploadedMetadataResults(); results.forEach((name, uploadedMetadata) -> { - if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class) && - uploadedMetadata.getComponent().contains(RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) { - response.uploadedIndicesRoutingMetadata.add((UploadedIndexMetadata) uploadedMetadata); - } else if (name.contains(CUSTOM_METADATA)) { + if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class) + && uploadedMetadata.getComponent().contains(RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) { + response.uploadedIndicesRoutingMetadata.add((UploadedIndexMetadata) uploadedMetadata); + } else if (name.contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom-- String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1]; response.uploadedCustomMetadataMap.put( diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java index 6f175dad190a1..c584e8400939f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java @@ -8,8 +8,6 @@ package org.opensearch.gateway.remote.routingtable; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java index c7fb5f1e27ccc..da620bc4eba2b 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java @@ -95,15 +95,14 @@ public enum PathType { public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.isNull(hashAlgorithm) : "hashAlgorithm is expected to be null with fixed remote store path type"; // Hash algorithm is not used in FIXED path type - BlobPath path = pathInput.basePath() - .add(pathInput.indexUUID()); - if(pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput)pathInput; + BlobPath path = pathInput.basePath().add(pathInput.indexUUID()); + if (pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; path.add(remoteStorePathInput.shardId()) .add(remoteStorePathInput.dataCategory().getName()) .add(remoteStorePathInput.dataType().getName()); } - return path; + return path; } @Override @@ -115,17 +114,17 @@ boolean requiresHashAlgorithm() { @Override public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - BlobPath path = BlobPath.cleanPath() + BlobPath path = BlobPath.cleanPath() .add(hashAlgorithm.hash(pathInput)) .add(pathInput.basePath()) .add(pathInput.indexUUID()); - if(pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput)pathInput; + if (pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; path.add(remoteStorePathInput.shardId()) .add(remoteStorePathInput.dataCategory().getName()) .add(remoteStorePathInput.dataType().getName()); } - return path; + return path; } @Override @@ -137,11 +136,9 @@ boolean requiresHashAlgorithm() { @Override public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - BlobPath path = pathInput.basePath() - .add(hashAlgorithm.hash(pathInput)) - .add(pathInput.indexUUID()); - if(pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput)pathInput; + BlobPath path = pathInput.basePath().add(hashAlgorithm.hash(pathInput)).add(pathInput.indexUUID()); + if (pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; path.add(remoteStorePathInput.shardId()) .add(remoteStorePathInput.dataCategory().getName()) .add(remoteStorePathInput.dataType().getName()); @@ -198,8 +195,8 @@ public static PathType fromCode(int code) { * @return the blob path for the path input. */ public BlobPath path(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { - if(pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput)pathInput; + if (pathInput instanceof RemoteStorePathInput) { + RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; DataCategory dataCategory = remoteStorePathInput.dataCategory(); DataType dataType = remoteStorePathInput.dataType(); assert dataCategory.isSupportedDataType(dataType) : "category:" @@ -243,9 +240,10 @@ public enum PathHashAlgorithm { @Override String hash(RemoteStorePathStrategy.BasePathInput pathInput) { String input = pathInput.indexUUID(); - if(pathInput instanceof RemoteStorePathInput) { + if (pathInput instanceof RemoteStorePathInput) { RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; - input += remoteStorePathInput.shardId() + remoteStorePathInput.dataCategory().getName() + remoteStorePathInput.dataType() + input += remoteStorePathInput.shardId() + remoteStorePathInput.dataCategory().getName() + remoteStorePathInput + .dataType() .getName(); } long hash = FNV1a.hash64(input); @@ -260,9 +258,10 @@ String hash(RemoteStorePathStrategy.BasePathInput pathInput) { @Override String hash(RemoteStorePathStrategy.BasePathInput pathInput) { String input = pathInput.indexUUID(); - if(pathInput instanceof RemoteStorePathInput) { + if (pathInput instanceof RemoteStorePathInput) { RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; - input += remoteStorePathInput.shardId() + remoteStorePathInput.dataCategory().getName() + remoteStorePathInput.dataType() + input += remoteStorePathInput.shardId() + remoteStorePathInput.dataCategory().getName() + remoteStorePathInput + .dataType() .getName(); } long hash = FNV1a.hash64(input); diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java index 252fcde2c4105..cf6d50d713bf8 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java @@ -134,7 +134,7 @@ public BasePathInput build() { */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class RemoteStorePathInput extends BasePathInput{ + public static class RemoteStorePathInput extends BasePathInput { private final String shardId; private final DataCategory dataCategory; private final DataType dataType; @@ -172,7 +172,7 @@ public static Builder builder() { */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class Builder extends BasePathInput.Builder{ + public static class Builder extends BasePathInput.Builder { private String shardId; private DataCategory dataCategory; private DataType dataType; diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 5d4c7280dba31..4221652f11c63 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -11,19 +11,12 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.DiffableUtils; -import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.RoutingTable; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.index.Index; -import org.opensearch.gateway.remote.RemoteClusterStateServiceTests; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; @@ -32,7 +25,6 @@ import org.junit.After; import org.junit.Before; -import java.util.Map; import java.util.function.Supplier; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; @@ -102,15 +94,25 @@ public void testGetChangedIndicesRouting() { RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).routingTable(routingTable).build(); - assertEquals(0, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), state.getRoutingTable()).getUpserts().size()); + assertEquals( + 0, + RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), state.getRoutingTable()).getUpserts().size() + ); - //Reversing order to check for equality without order. + // Reversing order to check for equality without order. IndexRoutingTable indexRouting = routingTable.getIndicesRouting().get(indexName); - IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(index).addShard(indexRouting.getShards().get(0).replicaShards().get(0)) - .addShard(indexRouting.getShards().get(0).primaryShard()).build(); - ClusterState newState = ClusterState.builder(ClusterName.DEFAULT).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build(); + IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(index) + .addShard(indexRouting.getShards().get(0).replicaShards().get(0)) + .addShard(indexRouting.getShards().get(0).primaryShard()) + .build(); + ClusterState newState = ClusterState.builder(ClusterName.DEFAULT) + .routingTable(RoutingTable.builder().add(indexRoutingTable).build()) + .build(); - assertEquals(0, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), newState.getRoutingTable()).getUpserts().size()); + assertEquals( + 0, + RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), newState.getRoutingTable()).getUpserts().size() + ); } } From a27b48dafc5bd3fb8612e30dda4faffe6a265fe9 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 5 Jun 2024 17:48:43 +0530 Subject: [PATCH 39/50] Add tests Signed-off-by: Himshikha Gupta --- .../remote/RemoteRoutingTableService.java | 17 +- .../opensearch/common/blobstore/BlobPath.java | 14 + .../remote/RemoteClusterStateService.java | 33 +- .../routing/IndexShardRoutingTableTests.java | 6 + .../RemoteRoutingTableServiceTests.java | 401 +++++++++++++++++- .../remote/ClusterMetadataManifestTests.java | 10 +- .../RemoteClusterStateServiceTests.java | 168 +++++++- 7 files changed, 611 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index b41d8456e40bc..5473a8039f9dc 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -48,7 +48,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -141,14 +140,14 @@ public CheckedRunnable getIndexRoutingAsyncAction( public List getAllUploadedIndicesRouting( ClusterMetadataManifest previousManifest, - List indicesRoutingToUpload, - Set indicesRoutingToDelete + List indicesRoutingUploaded, + List indicesRoutingToDelete ) { final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() .stream() .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); - indicesRoutingToUpload.forEach( + indicesRoutingUploaded.forEach( uploadedIndexRouting -> allUploadedIndicesRouting.put(uploadedIndexRouting.getIndexName(), uploadedIndexRouting) ); indicesRoutingToDelete.forEach(allUploadedIndicesRouting::remove); @@ -176,7 +175,8 @@ private void uploadIndex( blobContainer.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); completionListener.onResponse(null); } catch (IOException e) { - throw new IOException("Failed to write IndexRoutingTable to remote store. ", e); + logger.error("Failed to write IndexRoutingTable to remote store. ", e); + completionListener.onFailure(e); } return; } @@ -199,12 +199,13 @@ private void uploadIndex( completionListener ); } catch (IOException e) { - throw new IOException("Failed to write IndexRoutingTable to remote store. ", e); + logger.error("Failed to write IndexRoutingTable to remote store. ", e); + completionListener.onFailure(e); } } catch (IOException e) { - throw new IOException("Failed to create transfer object for IndexRoutingTable for remote store upload. ", e); + logger.error("Failed to create transfer object for IndexRoutingTable for remote store upload. ", e); + completionListener.onFailure(e); } - } private String getIndexRoutingFileName() { diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java b/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java index 6f3e8be7c28b8..79d6e6c5659d4 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java +++ b/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Objects; /** * The list of paths where a blob can reside. The contents of the paths are dependent upon the implementation of {@link BlobContainer}. @@ -110,6 +111,19 @@ public BlobPath parent() { } } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BlobPath strings = (BlobPath) o; + return Objects.equals(paths, strings.paths); + } + + @Override + public int hashCode() { + return Objects.hashCode(paths); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 920198090756c..400258c13f15a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -26,6 +26,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; @@ -212,7 +213,7 @@ public class RemoteClusterStateService implements Closeable { private final List indexMetadataUploadListeners; private BlobStoreRepository blobStoreRepository; private BlobStoreTransferService blobStoreTransferService; - private Optional remoteRoutingTableService; + private final Optional remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; private volatile TimeValue indexMetadataUploadTimeout; @@ -298,7 +299,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri true, true, true, - new ArrayList<>(clusterState.getRoutingTable().indicesRouting().values()) + remoteRoutingTableService.isPresent() ? new ArrayList<>(clusterState.getRoutingTable().indicesRouting().values()) : List.of() ); final ClusterMetadataManifest manifest = uploadManifest( clusterState, @@ -316,16 +317,19 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri remoteStateStats.stateTook(durationMillis); if (durationMillis >= slowWriteLoggingThreshold.getMillis()) { logger.warn( - "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " + "wrote full state with [{}] indices", + "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " + + "wrote full state with [{}] indices and [{}] indicesRouting", durationMillis, slowWriteLoggingThreshold, - uploadedMetadataResults.uploadedIndexMetadata.size() + uploadedMetadataResults.uploadedIndexMetadata.size(), + uploadedMetadataResults.uploadedIndicesRoutingMetadata.size() ); } else { logger.info( - "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices and global metadata", + "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices, [{}] indicesRouting and global metadata", durationMillis, - uploadedMetadataResults.uploadedIndexMetadata.size() + uploadedMetadataResults.uploadedIndexMetadata.size(), + uploadedMetadataResults.uploadedIndicesRoutingMetadata.size() ); } return manifest; @@ -392,9 +396,12 @@ public ClusterMetadataManifest writeIncrementalMetadata( } List indicesRoutingToUpload = new ArrayList<>(); + DiffableUtils.MapDiff> routingTableDiff = null; if (remoteRoutingTableService.isPresent()) { - DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService - .getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable()); + routingTableDiff = RemoteRoutingTableService.getIndicesRoutingMapDiff( + previousClusterState.getRoutingTable(), + clusterState.getRoutingTable() + ); routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v)); } @@ -436,7 +443,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( .getAllUploadedIndicesRouting( previousManifest, uploadedMetadataResults.uploadedIndicesRoutingMetadata, - indicesToBeDeletedFromRemote.keySet() + routingTableDiff.getDeletes() ); } @@ -570,7 +577,7 @@ private UploadedMetadataResults writeMetadataInParallel( indicesRoutingToUpload.forEach(indexRoutingTable -> { try { uploadTasks.put( - indexRoutingTable.getIndex().getName() + "--indexRouting", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName(), remoteRoutingTableService.get() .getIndexRoutingAsyncAction( clusterState, @@ -580,7 +587,7 @@ private UploadedMetadataResults writeMetadataInParallel( ) ); } catch (IOException e) { - e.printStackTrace(); + throw new RemoteStateTransferException("Failed to create upload tasks for index routing table", e); } }); @@ -811,9 +818,7 @@ public void close() throws IOException { if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } - if (this.remoteRoutingTableService.isPresent()) { - this.remoteRoutingTableService.get().close(); - } + this.remoteRoutingTableService.ifPresent(AbstractLifecycleComponent::close); } public void start() { diff --git a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java index e881016fb9305..6bfe60980adf3 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java @@ -69,6 +69,12 @@ public void testEquals() { assertNotEquals(table1, null); assertNotEquals(table1, s); assertNotEquals(table1, table3); + + ShardRouting primary = TestShardRouting.newShardRouting(shardId, "node-1", true, ShardRoutingState.STARTED); + ShardRouting replica = TestShardRouting.newShardRouting(shardId, "node-2", false, ShardRoutingState.STARTED); + IndexShardRoutingTable table4 = new IndexShardRoutingTable(shardId, Arrays.asList(primary, replica)); + IndexShardRoutingTable table5 = new IndexShardRoutingTable(shardId, Arrays.asList(replica, primary)); + assertEquals(table4, table5); } public void testShardsMatchingPredicate() { diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 4221652f11c63..ee903f4ffff5a 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -9,28 +9,60 @@ package org.opensearch.cluster.routing.remote; import org.opensearch.Version; +import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; +import org.mockito.ArgumentCaptor; + +import static org.opensearch.cluster.routing.remote.RemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; +import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { @@ -39,6 +71,9 @@ public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { private Supplier repositoriesServiceSupplier; private RepositoriesService repositoriesService; private BlobStoreRepository blobStoreRepository; + private BlobStore blobStore; + private BlobContainer blobContainer; + private BlobPath basePath; @Before public void setup() { @@ -48,14 +83,19 @@ public void setup() { Settings settings = Settings.builder() .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .put(FsRepository.REPOSITORIES_COMPRESS_SETTING.getKey(), false) .build(); - blobStoreRepository = mock(BlobStoreRepository.class); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + blobStore = mock(BlobStore.class); + blobContainer = mock(BlobContainer.class); when(repositoriesService.repository("routing_repository")).thenReturn(blobStoreRepository); - + when(blobStoreRepository.blobStore()).thenReturn(blobStore); Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); FeatureFlags.initializeFeatureFlags(nodeSettings); + basePath = BlobPath.cleanPath().add("base-path"); + remoteRoutingTableService = new RemoteRoutingTableService(repositoriesServiceSupplier, settings); } @@ -81,7 +121,7 @@ public void testFailStartWhenNotBlobRepository() { assertThrows(AssertionError.class, () -> remoteRoutingTableService.start()); } - public void testGetChangedIndicesRouting() { + public void testGetIndicesRoutingMapDiff() { String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); final Index index = new Index(indexName, "uuid"); final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( @@ -92,27 +132,360 @@ public void testGetChangedIndicesRouting() { ).numberOfShards(1).numberOfReplicas(1).build(); RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).routingTable(routingTable).build(); - assertEquals( - 0, - RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), state.getRoutingTable()).getUpserts().size() - ); + DiffableUtils.MapDiff> diff = RemoteRoutingTableService + .getIndicesRoutingMapDiff(routingTable, routingTable); + assertEquals(0, diff.getUpserts().size()); + assertEquals(0, diff.getDeletes().size()); // Reversing order to check for equality without order. IndexRoutingTable indexRouting = routingTable.getIndicesRouting().get(indexName); - IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(index) + IndexRoutingTable indexRoutingTableReversed = IndexRoutingTable.builder(index) .addShard(indexRouting.getShards().get(0).replicaShards().get(0)) .addShard(indexRouting.getShards().get(0).primaryShard()) .build(); - ClusterState newState = ClusterState.builder(ClusterName.DEFAULT) - .routingTable(RoutingTable.builder().add(indexRoutingTable).build()) + RoutingTable routingTable2 = RoutingTable.builder().add(indexRoutingTableReversed).build(); + + diff = RemoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(0, diff.getUpserts().size()); + assertEquals(0, diff.getDeletes().size()); + } + + public void testGetIndicesRoutingMapDiffIndexAdded() { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build(); + RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); + + String indexName2 = randomAlphaOfLength(randomIntBetween(1, 50)); + int noOfShards = randomInt(1000); + int noOfReplicas = randomInt(10); + final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(indexName2).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid2") + .build() + ).numberOfShards(noOfShards).numberOfReplicas(noOfReplicas).build(); + RoutingTable routingTable2 = RoutingTable.builder(routingTable).addAsNew(indexMetadata2).build(); + + DiffableUtils.MapDiff> diff = RemoteRoutingTableService + .getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(1, diff.getUpserts().size()); + assertNotNull(diff.getUpserts().get(indexName2)); + assertEquals(noOfShards, diff.getUpserts().get(indexName2).getShards().size()); + + assertEquals(0, diff.getDeletes().size()); + } + + public void testGetIndicesRoutingMapDiffShardChanged() { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + final Index index = new Index(indexName, "uuid"); + int noOfShards = randomInt(1000); + int noOfReplicas = randomInt(10); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(noOfShards).numberOfReplicas(noOfReplicas).build(); + + RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); + + final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(noOfShards + 1).numberOfReplicas(noOfReplicas).build(); + RoutingTable routingTable2 = RoutingTable.builder().addAsNew(indexMetadata2).build(); + + DiffableUtils.MapDiff> diff = RemoteRoutingTableService + .getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(1, diff.getUpserts().size()); + assertNotNull(diff.getUpserts().get(indexName)); + assertEquals(noOfShards + 1, diff.getUpserts().get(indexName).getShards().size()); + assertEquals(noOfReplicas + 1, diff.getUpserts().get(indexName).getShards().get(0).getSize()); + assertEquals(0, diff.getDeletes().size()); + + final IndexMetadata indexMetadata3 = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(noOfShards + 1).numberOfReplicas(noOfReplicas + 1).build(); + RoutingTable routingTable3 = RoutingTable.builder().addAsNew(indexMetadata3).build(); + + diff = RemoteRoutingTableService.getIndicesRoutingMapDiff(routingTable2, routingTable3); + assertEquals(1, diff.getUpserts().size()); + assertNotNull(diff.getUpserts().get(indexName)); + assertEquals(noOfShards + 1, diff.getUpserts().get(indexName).getShards().size()); + assertEquals(noOfReplicas + 2, diff.getUpserts().get(indexName).getShards().get(0).getSize()); + + assertEquals(0, diff.getDeletes().size()); + } + + public void testGetIndicesRoutingMapDiffIndexDeleted() { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build(); + RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); + + String indexName2 = randomAlphaOfLength(randomIntBetween(1, 50)); + final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(indexName2).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid2") + .build() + ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build(); + RoutingTable routingTable2 = RoutingTable.builder().addAsNew(indexMetadata2).build(); + + DiffableUtils.MapDiff> diff = RemoteRoutingTableService + .getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(1, diff.getUpserts().size()); + assertNotNull(diff.getUpserts().get(indexName2)); + + assertEquals(1, diff.getDeletes().size()); + assertEquals(indexName, diff.getDeletes().get(0)); + } + + public void testGetIndexRoutingAsyncAction() throws IOException { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + ClusterState clusterState = createClusterState(indexName); + BlobPath expectedPath = getPath(); + + LatchedActionListener listener = mock(LatchedActionListener.class); + when(blobStore.blobContainer(expectedPath)).thenReturn(blobContainer); + + remoteRoutingTableService.start(); + CheckedRunnable runnable = remoteRoutingTableService.getIndexRoutingAsyncAction( + clusterState, + clusterState.routingTable().getIndicesRouting().get(indexName), + listener, + basePath + ); + assertNotNull(runnable); + runnable.run(); + + verify(blobContainer, times(1)).writeBlob(anyString(), any(StreamInput.class), anyLong(), eq(true)); + verify(listener, times(1)).onResponse(any(ClusterMetadataManifest.UploadedMetadata.class)); + } + + public void testGetIndexRoutingAsyncActionFailureInBlobRepo() throws IOException { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + ClusterState clusterState = createClusterState(indexName); + BlobPath expectedPath = getPath(); + + LatchedActionListener listener = mock(LatchedActionListener.class); + when(blobStore.blobContainer(expectedPath)).thenReturn(blobContainer); + doThrow(new IOException("testing failure")).when(blobContainer).writeBlob(anyString(), any(StreamInput.class), anyLong(), eq(true)); + + remoteRoutingTableService.start(); + CheckedRunnable runnable = remoteRoutingTableService.getIndexRoutingAsyncAction( + clusterState, + clusterState.routingTable().getIndicesRouting().get(indexName), + listener, + basePath + ); + assertNotNull(runnable); + runnable.run(); + + verify(blobContainer, times(1)).writeBlob(anyString(), any(StreamInput.class), anyLong(), eq(true)); + verify(listener, times(1)).onFailure(any(RemoteClusterStateService.RemoteStateTransferException.class)); + } + + public void testGetIndexRoutingAsyncActionAsyncRepo() throws IOException { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + ClusterState clusterState = createClusterState(indexName); + BlobPath expectedPath = getPath(); + + LatchedActionListener listener = mock(LatchedActionListener.class); + blobContainer = mock(AsyncMultiStreamBlobContainer.class); + when(blobStore.blobContainer(expectedPath)).thenReturn(blobContainer); + ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class); + ArgumentCaptor writeContextArgumentCaptor = ArgumentCaptor.forClass(WriteContext.class); + ConcurrentHashMap capturedWriteContext = new ConcurrentHashMap<>(); + + doAnswer((i) -> { + actionListenerArgumentCaptor.getValue().onResponse(null); + WriteContext writeContext = writeContextArgumentCaptor.getValue(); + capturedWriteContext.put(writeContext.getFileName().split(DELIMITER)[0], writeContextArgumentCaptor.getValue()); + return null; + }).when((AsyncMultiStreamBlobContainer) blobContainer) + .asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture()); + + remoteRoutingTableService.start(); + CheckedRunnable runnable = remoteRoutingTableService.getIndexRoutingAsyncAction( + clusterState, + clusterState.routingTable().getIndicesRouting().get(indexName), + listener, + basePath + ); + assertNotNull(runnable); + runnable.run(); + + assertEquals(1, actionListenerArgumentCaptor.getAllValues().size()); + assertEquals(1, writeContextArgumentCaptor.getAllValues().size()); + assertNotNull(capturedWriteContext.get("index_routing")); + assertEquals(capturedWriteContext.get("index_routing").getWritePriority(), WritePriority.URGENT); + assertTrue(capturedWriteContext.get("index_routing").getFileName().startsWith(RemoteRoutingTableService.INDEX_ROUTING_FILE_PREFIX)); + } + + public void testGetIndexRoutingAsyncActionAsyncRepoFailureInRepo() throws IOException { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + ClusterState clusterState = createClusterState(indexName); + BlobPath expectedPath = getPath(); + + LatchedActionListener listener = mock(LatchedActionListener.class); + blobContainer = mock(AsyncMultiStreamBlobContainer.class); + when(blobStore.blobContainer(expectedPath)).thenReturn(blobContainer); + + doThrow(new IOException("Testing failure")).when((AsyncMultiStreamBlobContainer) blobContainer) + .asyncBlobUpload(any(WriteContext.class), any(ActionListener.class)); + + remoteRoutingTableService.start(); + CheckedRunnable runnable = remoteRoutingTableService.getIndexRoutingAsyncAction( + clusterState, + clusterState.routingTable().getIndicesRouting().get(indexName), + listener, + basePath + ); + assertNotNull(runnable); + runnable.run(); + verify(listener, times(1)).onFailure(any(RemoteClusterStateService.RemoteStateTransferException.class)); + } + + public void testGetAllUploadedIndicesRouting() { + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().build(); + final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata = new ClusterMetadataManifest.UploadedIndexMetadata( + "test-index", + "index-uuid", + "index-filename", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + ); + + List allIndiceRoutingMetadata = remoteRoutingTableService + .getAllUploadedIndicesRouting(previousManifest, List.of(uploadedIndexMetadata), List.of()); + assertNotNull(allIndiceRoutingMetadata); + assertEquals(1, allIndiceRoutingMetadata.size()); + assertEquals(uploadedIndexMetadata, allIndiceRoutingMetadata.get(0)); + } + + public void testGetAllUploadedIndicesRoutingExistingIndexInManifest() { + final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata = new ClusterMetadataManifest.UploadedIndexMetadata( + "test-index", + "index-uuid", + "index-filename", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() + .indicesRouting(List.of(uploadedIndexMetadata)) + .build(); + + List allIndiceRoutingMetadata = remoteRoutingTableService + .getAllUploadedIndicesRouting(previousManifest, List.of(uploadedIndexMetadata), List.of()); + assertNotNull(allIndiceRoutingMetadata); + assertEquals(1, allIndiceRoutingMetadata.size()); + assertEquals(uploadedIndexMetadata, allIndiceRoutingMetadata.get(0)); + } + + public void testGetAllUploadedIndicesRoutingNewIndexFromManifest() { + final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata = new ClusterMetadataManifest.UploadedIndexMetadata( + "test-index", + "index-uuid", + "index-filename", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() + .indicesRouting(List.of(uploadedIndexMetadata)) + .build(); + final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata( + "test-index2", + "index-uuid", + "index-filename", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + ); + + List allIndiceRoutingMetadata = remoteRoutingTableService + .getAllUploadedIndicesRouting(previousManifest, List.of(uploadedIndexMetadata2), List.of()); + assertNotNull(allIndiceRoutingMetadata); + assertEquals(2, allIndiceRoutingMetadata.size()); + assertEquals(uploadedIndexMetadata, allIndiceRoutingMetadata.get(0)); + assertEquals(uploadedIndexMetadata2, allIndiceRoutingMetadata.get(1)); + } + + public void testGetAllUploadedIndicesRoutingIndexDeleted() { + final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata = new ClusterMetadataManifest.UploadedIndexMetadata( + "test-index", + "index-uuid", + "index-filename", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata( + "test-index2", + "index-uuid", + "index-filename", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() + .indicesRouting(List.of(uploadedIndexMetadata, uploadedIndexMetadata2)) .build(); - assertEquals( - 0, - RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), newState.getRoutingTable()).getUpserts().size() + List allIndiceRoutingMetadata = remoteRoutingTableService + .getAllUploadedIndicesRouting(previousManifest, List.of(uploadedIndexMetadata2), List.of("test-index")); + assertNotNull(allIndiceRoutingMetadata); + assertEquals(1, allIndiceRoutingMetadata.size()); + assertEquals(uploadedIndexMetadata2, allIndiceRoutingMetadata.get(0)); + } + + public void testGetAllUploadedIndicesRoutingNoChange() { + final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata = new ClusterMetadataManifest.UploadedIndexMetadata( + "test-index", + "index-uuid", + "index-filename", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); + final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata( + "test-index2", + "index-uuid", + "index-filename", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() + .indicesRouting(List.of(uploadedIndexMetadata, uploadedIndexMetadata2)) + .build(); + + List allIndiceRoutingMetadata = remoteRoutingTableService + .getAllUploadedIndicesRouting(previousManifest, List.of(), List.of()); + assertNotNull(allIndiceRoutingMetadata); + assertEquals(2, allIndiceRoutingMetadata.size()); + assertEquals(uploadedIndexMetadata, allIndiceRoutingMetadata.get(0)); + assertEquals(uploadedIndexMetadata2, allIndiceRoutingMetadata.get(1)); } + private ClusterState createClusterState(String indexName) { + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build(); + RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); + return ClusterState.builder(ClusterName.DEFAULT).routingTable(routingTable).build(); + } + + private BlobPath getPath() { + BlobPath indexRoutingPath = basePath.add(INDEX_ROUTING_PATH_TOKEN); + return RemoteStoreEnums.PathType.HASHED_PREFIX.path( + RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID("uuid").build(), + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64 + ); + } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index d1f559eb75f85..1e911476a33e6 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -12,6 +12,7 @@ import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -366,6 +367,13 @@ public void testClusterMetadataManifestXContentV2() throws IOException { public void testClusterMetadataManifestXContentV3() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); + UploadedIndexMetadata uploadedIndexRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "test-uuid", + "routing-path", + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + ); + ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( 1L, 1L, @@ -400,7 +408,7 @@ public void testClusterMetadataManifestXContentV3() throws IOException { ) ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), 1L, - Collections.singletonList(uploadedIndexMetadata) + Collections.singletonList(uploadedIndexRoutingMetadata) ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 03694f24bf2c1..408f3c554dbe8 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -19,6 +19,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -163,6 +164,8 @@ public void setup() { blobStore = mock(BlobStore.class); when(blobStoreRepository.blobStore()).thenReturn(blobStore); when(repositoriesService.repository("remote_store_repository")).thenReturn(blobStoreRepository); + when(repositoriesService.repository("routing_repository")).thenReturn(blobStoreRepository); + when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(xContentRegistry); remoteClusterStateService = new RemoteClusterStateService( "test-node-id", @@ -1528,6 +1531,168 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { assertTrue(remoteClusterStateService.getRemoteRoutingTableService().isPresent()); } + public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { + initializeRoutingTable(); + + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + mockBlobStoreObjects(); + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + "indexRouting--" + ); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1L) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + } + + public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOException { + initializeRoutingTable(); + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + AsyncMultiStreamBlobContainer container = (AsyncMultiStreamBlobContainer) mockBlobStoreObjects(AsyncMultiStreamBlobContainer.class); + + ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class); + ArgumentCaptor writeContextArgumentCaptor = ArgumentCaptor.forClass(WriteContext.class); + ConcurrentHashMap capturedWriteContext = new ConcurrentHashMap<>(); + doAnswer((i) -> { + actionListenerArgumentCaptor.getValue().onResponse(null); + WriteContext writeContext = writeContextArgumentCaptor.getValue(); + capturedWriteContext.put(writeContext.getFileName().split(DELIMITER)[0], writeContextArgumentCaptor.getValue()); + return null; + }).when(container).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture()); + + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); + + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); + + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + "indexRouting--" + ); + + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName())); + assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID())); + assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue()); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + + assertEquals(8, actionListenerArgumentCaptor.getAllValues().size()); + assertEquals(8, writeContextArgumentCaptor.getAllValues().size()); + } + + public void testWriteIncrementalMetadataSuccessWithRoutingTable() throws IOException { + initializeRoutingTable(); + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + mockBlobStoreObjects(); + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); + final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) + .build(); + + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(Collections.emptyList()).build(); + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); + + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata( + previousClusterState, + clusterState, + previousManifest + ); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + "indexRouting--" + ); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + } + + private void initializeRoutingTable() { + Settings newSettings = Settings.builder() + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + clusterSettings.applySettings(newSettings); + + Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + remoteClusterStateService = new RemoteClusterStateService( + "test-node-id", + repositoriesServiceSupplier, + newSettings, + clusterSettings, + () -> 0L, + threadPool, + List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)) + ); + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } @@ -1950,7 +2115,8 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() { .templates(templatesMetadata) .putCustom(customMetadata1.getWriteableName(), customMetadata1) .build() - ); + ) + .routingTable(RoutingTable.builder().addAsNew(indexMetadata).version(1L).build()); } private static DiscoveryNodes nodesWithLocalNodeClusterManager() { From 089d4ee3ded0e002e1c6fae1bc245f1771a40730 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 6 Jun 2024 18:15:13 +0530 Subject: [PATCH 40/50] rebasing Signed-off-by: Himshikha Gupta --- .../remote/RemoteClusterStateService.java | 2 +- .../RemoteRoutingTableServiceTests.java | 24 +++++++++++++++++++ .../RemoteClusterStateServiceTests.java | 12 ++++++---- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 666500e1f123a..f828780d2fa57 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -828,8 +828,8 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; - this.remoteRoutingTableService.ifPresent(RemoteRoutingTableService::start); remoteClusterStateCleanupManager.start(); + this.remoteRoutingTableService.ifPresent(RemoteRoutingTableService::start); } private ClusterMetadataManifest uploadManifest( diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 10c17a72bd539..88e4224117d49 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -230,6 +230,30 @@ public void testGetIndicesRoutingMapDiffShardChanged() { assertEquals(0, diff.getDeletes().size()); } + public void testGetIndicesRoutingMapDiffShardDetailChanged() { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + final Index index = new Index(indexName, "uuid"); + int noOfShards = randomInt(1000); + int noOfReplicas = randomInt(10); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(noOfShards).numberOfReplicas(noOfReplicas).build(); + + RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); + RoutingTable routingTable2 = RoutingTable.builder().addAsRecovery(indexMetadata).build(); + + DiffableUtils.MapDiff> diff = RemoteRoutingTableService + .getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(1, diff.getUpserts().size()); + assertNotNull(diff.getUpserts().get(indexName)); + assertEquals(noOfShards, diff.getUpserts().get(indexName).getShards().size()); + assertEquals(noOfReplicas +1, diff.getUpserts().get(indexName).getShards().get(0).getSize()); + assertEquals(0, diff.getDeletes().size()); + } + public void testGetIndicesRoutingMapDiffIndexDeleted() { String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index d7d96a07170e7..5efc89e816aa7 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -20,6 +20,7 @@ import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; @@ -1415,9 +1416,10 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { initializeRoutingTable(); + mockBlobStoreObjects(); + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); - mockBlobStoreObjects(); remoteClusterStateService.start(); final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); @@ -1425,7 +1427,7 @@ public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { "test-index", "index-uuid", "routing-filename", - "indexRouting--" + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() .indices(List.of(uploadedIndexMetadata)) @@ -1475,7 +1477,7 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx "test-index", "index-uuid", "routing-filename", - "indexRouting--" + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() @@ -1530,7 +1532,7 @@ public void testWriteIncrementalMetadataSuccessWithRoutingTable() throws IOExcep "test-index", "index-uuid", "routing-filename", - "indexRouting--" + RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() .indices(List.of(uploadedIndexMetadata)) @@ -1568,7 +1570,7 @@ private void initializeRoutingTable() { "test-node-id", repositoriesServiceSupplier, newSettings, - clusterSettings, + clusterService, () -> 0L, threadPool, List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)) From 41a5f647769d1ac868d8f8a8ac943e897ca3500a Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 6 Jun 2024 23:22:46 +0530 Subject: [PATCH 41/50] spotless fix Signed-off-by: Himshikha Gupta --- .../opensearch/gateway/remote/RemoteClusterStateService.java | 2 -- .../cluster/routing/remote/RemoteRoutingTableServiceTests.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index f828780d2fa57..7caf655f887eb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -21,7 +21,6 @@ import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -29,7 +28,6 @@ import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; -import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 88e4224117d49..ac48e0273784d 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -250,7 +250,7 @@ public void testGetIndicesRoutingMapDiffShardDetailChanged() { assertEquals(1, diff.getUpserts().size()); assertNotNull(diff.getUpserts().get(indexName)); assertEquals(noOfShards, diff.getUpserts().get(indexName).getShards().size()); - assertEquals(noOfReplicas +1, diff.getUpserts().get(indexName).getShards().get(0).getSize()); + assertEquals(noOfReplicas + 1, diff.getUpserts().get(indexName).getShards().get(0).getSize()); assertEquals(0, diff.getDeletes().size()); } From df1704f8a4f7ae8282ae87a797b18ccab16c34b6 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 7 Jun 2024 13:53:06 +0530 Subject: [PATCH 42/50] Move path generation logic to path input Signed-off-by: Himshikha Gupta --- CHANGELOG.md | 1 + .../routing/IndexShardRoutingTable.java | 4 +- .../remote/RemoteRoutingTableService.java | 85 +++++++++- .../opensearch/common/blobstore/BlobPath.java | 4 +- .../common/settings/ClusterSettings.java | 3 + .../remote/RemoteClusterStateService.java | 2 +- .../index/remote/RemoteIndexPath.java | 4 +- .../index/remote/RemoteStoreEnums.java | 64 ++------ .../index/remote/RemoteStorePathStrategy.java | 29 +++- .../RemoteSegmentStoreDirectoryFactory.java | 4 +- .../RemoteStoreLockManagerFactory.java | 2 +- .../index/translog/RemoteFsTranslog.java | 4 +- .../RemoteRoutingTableServiceTests.java | 52 +++++- .../index/remote/RemoteStoreEnumsTests.java | 148 +++++------------- .../remote/RemoteStorePathStrategyTests.java | 87 ++++++++++ .../opensearch/test/OpenSearchTestCase.java | 2 +- 16 files changed, 297 insertions(+), 198 deletions(-) create mode 100644 server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 759cf53adf8b8..69c156e799111 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote State] Add async remote state deletion task running on an interval, configurable by a setting ([#13131](https://github.com/opensearch-project/OpenSearch/pull/13131)) - Allow setting query parameters on requests ([#13776](https://github.com/opensearch-project/OpenSearch/issues/13776)) - Add remote routing table for remote state publication with experimental feature flag ([#13304](https://github.com/opensearch-project/OpenSearch/pull/13304)) +- Add upload flow for writing routing table to remote store ([#13870](https://github.com/opensearch-project/OpenSearch/pull/13870)) ### Dependencies - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559)) diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 39589f46f7a4e..479143fa9a2f0 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -738,9 +738,7 @@ public boolean equals(Object o) { IndexShardRoutingTable that = (IndexShardRoutingTable) o; if (!shardId.equals(that.shardId)) return false; - if (!new HashSet<>(shards).equals(new HashSet<>(that.shards))) return false; - - return true; + return shards.size() == that.shards.size() && shards.containsAll(that.shards) && that.shards.containsAll(shards); } @Override diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 5473a8039f9dc..d5d55f841df16 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -26,6 +26,8 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.lucene.store.ByteArrayIndexInput; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; @@ -61,6 +63,30 @@ */ public class RemoteRoutingTableService extends AbstractLifecycleComponent { + /** + * This setting is used to set the remote routing table store blob store path type strategy. + */ + public static final Setting REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING = new Setting<>( + "cluster.remote_store.routing_table.path_type", + RemoteStoreEnums.PathType.HASHED_PREFIX.toString(), + RemoteStoreEnums.PathType::parseString, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting is used to set the remote routing table store blob store path hash algorithm strategy. + * This setting will come to effect if the {@link #REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING} + * is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}. + */ + public static final Setting REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING = new Setting<>( + "cluster.remote_store.routing_table.path_hash_algo", + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64.toString(), + RemoteStoreEnums.PathHashAlgorithm::parseString, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public static final String INDEX_ROUTING_PATH_TOKEN = "index-routing"; public static final String INDEX_ROUTING_FILE_PREFIX = "index_routing"; public static final String DELIMITER = "__"; @@ -70,11 +96,29 @@ public class RemoteRoutingTableService extends AbstractLifecycleComponent { private final Settings settings; private final Supplier repositoriesService; private BlobStoreRepository blobStoreRepository; + private RemoteStoreEnums.PathType pathType; + private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo; - public RemoteRoutingTableService(Supplier repositoriesService, Settings settings) { + public RemoteRoutingTableService( + Supplier repositoriesService, + Settings settings, + ClusterSettings clusterSettings + ) { assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; + this.pathType = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING); + this.pathHashAlgo = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING); + clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, this::setPathTypeSetting); + clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, this::setPathHashAlgoSetting); + } + + private void setPathTypeSetting(RemoteStoreEnums.PathType pathType) { + this.pathType = pathType; + } + + private void setPathHashAlgoSetting(RemoteStoreEnums.PathHashAlgorithm pathHashAlgo) { + this.pathHashAlgo = pathHashAlgo; } private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = @@ -90,6 +134,12 @@ public IndexRoutingTable read(StreamInput in, String key) throws IOException { } }; + /** + * Returns diff between the two routing tables, which includes upserts and deletes. + * @param before previous routing table + * @param after current routing table + * @return diff of the previous and current routing table + */ public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff( RoutingTable before, RoutingTable after @@ -102,6 +152,15 @@ public static DiffableUtils.MapDiff getIndexRoutingAsyncAction( ClusterState clusterState, IndexRoutingTable indexRouting, @@ -110,13 +169,13 @@ public CheckedRunnable getIndexRoutingAsyncAction( ) throws IOException { BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN); - BlobPath path = RemoteStoreEnums.PathType.HASHED_PREFIX.path( + BlobPath path = pathType.path( RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(), - RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64 + pathHashAlgo ); final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); - final String fileName = getIndexRoutingFileName(); + final String fileName = getIndexRoutingFileName(clusterState.term(), clusterState.version()); ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse( @@ -138,6 +197,13 @@ public CheckedRunnable getIndexRoutingAsyncAction( return () -> uploadIndex(indexRouting, fileName, blobContainer, completionListener); } + /** + * Combines IndicesRoutingMetadata from previous manifest and current uploaded indices, removes deleted indices. + * @param previousManifest previous manifest, used to get all existing indices routing paths + * @param indicesRoutingUploaded current uploaded indices routings + * @param indicesRoutingToDelete indices to delete + * @return combined list of metadata + */ public List getAllUploadedIndicesRouting( ClusterMetadataManifest previousManifest, List indicesRoutingUploaded, @@ -208,9 +274,14 @@ private void uploadIndex( } } - private String getIndexRoutingFileName() { - return String.join(DELIMITER, INDEX_ROUTING_FILE_PREFIX, RemoteStoreUtils.invertLong(System.currentTimeMillis())); - + private String getIndexRoutingFileName(long term, long version) { + return String.join( + DELIMITER, + INDEX_ROUTING_FILE_PREFIX, + RemoteStoreUtils.invertLong(term), + RemoteStoreUtils.invertLong(version), + RemoteStoreUtils.invertLong(System.currentTimeMillis()) + ); } @Override diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java b/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java index 79d6e6c5659d4..68af77714a319 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java +++ b/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java @@ -115,8 +115,8 @@ public BlobPath parent() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - BlobPath strings = (BlobPath) o; - return Objects.equals(paths, strings.paths); + BlobPath that = (BlobPath) o; + return Objects.equals(paths, that.paths); } @Override diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 297fc98764d07..1eeb8142ff5c8 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -77,6 +77,7 @@ import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterManagerService; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; @@ -721,6 +722,8 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING, + RemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, + RemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, // Admission Control Settings AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 7caf655f887eb..9b9b438619b82 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -265,7 +265,7 @@ public RemoteClusterStateService( this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService); this.indexMetadataUploadListeners = indexMetadataUploadListeners; this.remoteRoutingTableService = isRemoteRoutingTableEnabled(settings) - ? Optional.of(new RemoteRoutingTableService(repositoriesService, settings)) + ? Optional.of(new RemoteRoutingTableService(repositoriesService, settings, clusterSettings)) : Optional.empty(); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java index 3629fcfa79420..89b642b79df86 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java @@ -18,7 +18,7 @@ import org.opensearch.index.remote.RemoteStoreEnums.DataType; import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; -import org.opensearch.index.remote.RemoteStorePathStrategy.RemoteStorePathInput; +import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import java.io.IOException; import java.util.Collections; @@ -141,7 +141,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws DataCategory dataCategory = entry.getKey(); for (DataType type : entry.getValue()) { for (int shardNo = 0; shardNo < shardCount; shardNo++) { - RemoteStorePathInput pathInput = RemoteStorePathInput.builder() + PathInput pathInput = PathInput.builder() .basePath(new BlobPath().add(basePath)) .indexUUID(indexUUID) .shardId(Integer.toString(shardNo)) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java index da620bc4eba2b..ba33cef142094 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java @@ -13,7 +13,6 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.hash.FNV1a; -import org.opensearch.index.remote.RemoteStorePathStrategy.RemoteStorePathInput; import java.util.HashMap; import java.util.Locale; @@ -95,14 +94,7 @@ public enum PathType { public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.isNull(hashAlgorithm) : "hashAlgorithm is expected to be null with fixed remote store path type"; // Hash algorithm is not used in FIXED path type - BlobPath path = pathInput.basePath().add(pathInput.indexUUID()); - if (pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; - path.add(remoteStorePathInput.shardId()) - .add(remoteStorePathInput.dataCategory().getName()) - .add(remoteStorePathInput.dataType().getName()); - } - return path; + return pathInput.basePath().add(pathInput.variablePath()); } @Override @@ -114,17 +106,7 @@ boolean requiresHashAlgorithm() { @Override public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - BlobPath path = BlobPath.cleanPath() - .add(hashAlgorithm.hash(pathInput)) - .add(pathInput.basePath()) - .add(pathInput.indexUUID()); - if (pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; - path.add(remoteStorePathInput.shardId()) - .add(remoteStorePathInput.dataCategory().getName()) - .add(remoteStorePathInput.dataType().getName()); - } - return path; + return BlobPath.cleanPath().add(hashAlgorithm.hash(pathInput)).add(pathInput.basePath()).add(pathInput.variablePath()); } @Override @@ -136,14 +118,7 @@ boolean requiresHashAlgorithm() { @Override public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - BlobPath path = pathInput.basePath().add(hashAlgorithm.hash(pathInput)).add(pathInput.indexUUID()); - if (pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; - path.add(remoteStorePathInput.shardId()) - .add(remoteStorePathInput.dataCategory().getName()) - .add(remoteStorePathInput.dataType().getName()); - } - return path; + return pathInput.basePath().add(hashAlgorithm.hash(pathInput)).add(pathInput.variablePath()); } @Override @@ -195,16 +170,7 @@ public static PathType fromCode(int code) { * @return the blob path for the path input. */ public BlobPath path(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { - if (pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; - DataCategory dataCategory = remoteStorePathInput.dataCategory(); - DataType dataType = remoteStorePathInput.dataType(); - assert dataCategory.isSupportedDataType(dataType) : "category:" - + dataCategory - + " type:" - + dataType - + " are not supported together"; - } + return generatePath(pathInput, hashAlgorithm); } @@ -239,14 +205,11 @@ public enum PathHashAlgorithm { FNV_1A_BASE64(0) { @Override String hash(RemoteStorePathStrategy.BasePathInput pathInput) { - String input = pathInput.indexUUID(); - if (pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; - input += remoteStorePathInput.shardId() + remoteStorePathInput.dataCategory().getName() + remoteStorePathInput - .dataType() - .getName(); + StringBuilder input = new StringBuilder(); + for (String paths : pathInput.variablePath().toArray()) { + input.append(paths); } - long hash = FNV1a.hash64(input); + long hash = FNV1a.hash64(input.toString()); return longToUrlBase64(hash); } }, @@ -257,14 +220,11 @@ String hash(RemoteStorePathStrategy.BasePathInput pathInput) { FNV_1A_COMPOSITE_1(1) { @Override String hash(RemoteStorePathStrategy.BasePathInput pathInput) { - String input = pathInput.indexUUID(); - if (pathInput instanceof RemoteStorePathInput) { - RemoteStorePathInput remoteStorePathInput = (RemoteStorePathInput) pathInput; - input += remoteStorePathInput.shardId() + remoteStorePathInput.dataCategory().getName() + remoteStorePathInput - .dataType() - .getName(); + StringBuilder input = new StringBuilder(); + for (String paths : pathInput.variablePath().toArray()) { + input.append(paths); } - long hash = FNV1a.hash64(input); + long hash = FNV1a.hash64(input.toString()); return longToCompositeBase64AndBinaryEncoding(hash, 20); } }; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java index cf6d50d713bf8..6907e936efad2 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java @@ -67,7 +67,7 @@ public String toString() { return "RemoteStorePathStrategy{" + "type=" + type + ", hashAlgorithm=" + hashAlgorithm + '}'; } - public BlobPath generatePath(RemoteStorePathInput pathInput) { + public BlobPath generatePath(PathInput pathInput) { return type.path(pathInput, hashAlgorithm); } @@ -94,6 +94,10 @@ String indexUUID() { return indexUUID; } + BlobPath variablePath() { + return BlobPath.cleanPath().add(indexUUID); + } + /** * Returns a new builder for {@link BasePathInput}. */ @@ -134,16 +138,22 @@ public BasePathInput build() { */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class RemoteStorePathInput extends BasePathInput { + public static class PathInput extends BasePathInput { private final String shardId; private final DataCategory dataCategory; private final DataType dataType; - public RemoteStorePathInput(BlobPath basePath, String indexUUID, String shardId, DataCategory dataCategory, DataType dataType) { + public PathInput(BlobPath basePath, String indexUUID, String shardId, DataCategory dataCategory, DataType dataType) { super(basePath, indexUUID); this.shardId = Objects.requireNonNull(shardId); this.dataCategory = Objects.requireNonNull(dataCategory); this.dataType = Objects.requireNonNull(dataType); + assert dataCategory.isSupportedDataType(dataType) : "category:" + + dataCategory + + " type:" + + dataType + + " are not supported together"; + } String shardId() { @@ -158,15 +168,20 @@ DataType dataType() { return dataType; } + @Override + BlobPath variablePath() { + return super.variablePath().add(shardId).add(dataCategory.getName()).add(dataType.getName()); + } + /** - * Returns a new builder for {@link RemoteStorePathInput}. + * Returns a new builder for {@link PathInput}. */ public static Builder builder() { return new Builder(); } /** - * Builder for {@link RemoteStorePathInput}. + * Builder for {@link PathInput}. * * @opensearch.internal */ @@ -202,8 +217,8 @@ public Builder dataType(DataType dataType) { return this; } - public RemoteStorePathInput build() { - return new RemoteStorePathInput(super.basePath, super.indexUUID, shardId, dataCategory, dataType); + public PathInput build() { + return new PathInput(super.basePath, super.indexUUID, shardId, dataCategory, dataType); } } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index 72def0f2b2233..e462f6d4ac011 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -65,7 +65,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s BlobPath repositoryBasePath = blobStoreRepository.basePath(); String shardIdStr = String.valueOf(shardId.id()); - RemoteStorePathStrategy.RemoteStorePathInput dataPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() + RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardIdStr) @@ -80,7 +80,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s blobStoreRepository::maybeRateLimitRemoteDownloadTransfers ); - RemoteStorePathStrategy.RemoteStorePathInput mdPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() + RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardIdStr) diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java index 9fc118699caa5..45d466d3a8ce8 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java @@ -56,7 +56,7 @@ public static RemoteStoreMetadataLockManager newLockManager( assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobPath repositoryBasePath = ((BlobStoreRepository) repository).basePath(); - RemoteStorePathStrategy.RemoteStorePathInput lockFilesPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() + RemoteStorePathStrategy.PathInput lockFilesPathInput = RemoteStorePathStrategy.PathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardId) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 41b3ad5cc378f..67549c86b7dd2 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -304,7 +304,7 @@ public static TranslogTransferManager buildTranslogTransferManager( assert Objects.nonNull(pathStrategy); String indexUUID = shardId.getIndex().getUUID(); String shardIdStr = String.valueOf(shardId.id()); - RemoteStorePathStrategy.RemoteStorePathInput dataPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() + RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder() .basePath(blobStoreRepository.basePath()) .indexUUID(indexUUID) .shardId(shardIdStr) @@ -312,7 +312,7 @@ public static TranslogTransferManager buildTranslogTransferManager( .dataType(DATA) .build(); BlobPath dataPath = pathStrategy.generatePath(dataPathInput); - RemoteStorePathStrategy.RemoteStorePathInput mdPathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() + RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder() .basePath(blobStoreRepository.basePath()) .indexUUID(indexUUID) .shardId(shardIdStr) diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index ac48e0273784d..467e6cfa6941d 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -13,7 +13,9 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.CheckedRunnable; @@ -24,6 +26,7 @@ import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.action.ActionListener; @@ -33,6 +36,7 @@ import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.index.remote.RemoteStoreEnums; import org.opensearch.index.remote.RemoteStorePathStrategy; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; @@ -50,6 +54,7 @@ import org.mockito.ArgumentCaptor; +import static org.opensearch.cluster.routing.remote.RemoteRoutingTableService.INDEX_ROUTING_FILE_PREFIX; import static org.opensearch.cluster.routing.remote.RemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; @@ -58,6 +63,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.startsWith; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -98,7 +104,11 @@ public void setup() { basePath = BlobPath.cleanPath().add("base-path"); - remoteRoutingTableService = new RemoteRoutingTableService(repositoriesServiceSupplier, settings); + remoteRoutingTableService = new RemoteRoutingTableService( + repositoriesServiceSupplier, + settings, + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); } @After @@ -109,7 +119,14 @@ public void teardown() throws Exception { public void testFailInitializationWhenRemoteRoutingDisabled() { final Settings settings = Settings.builder().build(); - assertThrows(AssertionError.class, () -> new RemoteRoutingTableService(repositoriesServiceSupplier, settings)); + assertThrows( + AssertionError.class, + () -> new RemoteRoutingTableService( + repositoriesServiceSupplier, + settings, + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ) + ); } public void testFailStartWhenRepositoryNotSet() { @@ -300,7 +317,13 @@ public void testGetIndexRoutingAsyncAction() throws IOException { assertNotNull(runnable); runnable.run(); - verify(blobContainer, times(1)).writeBlob(anyString(), any(StreamInput.class), anyLong(), eq(true)); + String expectedFilePrefix = String.join( + DELIMITER, + INDEX_ROUTING_FILE_PREFIX, + RemoteStoreUtils.invertLong(clusterState.term()), + RemoteStoreUtils.invertLong(clusterState.version()) + ); + verify(blobContainer, times(1)).writeBlob(startsWith(expectedFilePrefix), any(StreamInput.class), anyLong(), eq(true)); verify(listener, times(1)).onResponse(any(ClusterMetadataManifest.UploadedMetadata.class)); } @@ -322,8 +345,13 @@ public void testGetIndexRoutingAsyncActionFailureInBlobRepo() throws IOException ); assertNotNull(runnable); runnable.run(); - - verify(blobContainer, times(1)).writeBlob(anyString(), any(StreamInput.class), anyLong(), eq(true)); + String expectedFilePrefix = String.join( + DELIMITER, + INDEX_ROUTING_FILE_PREFIX, + RemoteStoreUtils.invertLong(clusterState.term()), + RemoteStoreUtils.invertLong(clusterState.version()) + ); + verify(blobContainer, times(1)).writeBlob(startsWith(expectedFilePrefix), any(StreamInput.class), anyLong(), eq(true)); verify(listener, times(1)).onFailure(any(RemoteClusterStateService.RemoteStateTransferException.class)); } @@ -357,11 +385,17 @@ public void testGetIndexRoutingAsyncActionAsyncRepo() throws IOException { assertNotNull(runnable); runnable.run(); + String expectedFilePrefix = String.join( + DELIMITER, + INDEX_ROUTING_FILE_PREFIX, + RemoteStoreUtils.invertLong(clusterState.term()), + RemoteStoreUtils.invertLong(clusterState.version()) + ); assertEquals(1, actionListenerArgumentCaptor.getAllValues().size()); assertEquals(1, writeContextArgumentCaptor.getAllValues().size()); assertNotNull(capturedWriteContext.get("index_routing")); assertEquals(capturedWriteContext.get("index_routing").getWritePriority(), WritePriority.URGENT); - assertTrue(capturedWriteContext.get("index_routing").getFileName().startsWith(RemoteRoutingTableService.INDEX_ROUTING_FILE_PREFIX)); + assertTrue(capturedWriteContext.get("index_routing").getFileName().startsWith(expectedFilePrefix)); } public void testGetIndexRoutingAsyncActionAsyncRepoFailureInRepo() throws IOException { @@ -504,7 +538,11 @@ private ClusterState createClusterState(String indexName) { .build() ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build(); RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); - return ClusterState.builder(ClusterName.DEFAULT).routingTable(routingTable).build(); + return ClusterState.builder(ClusterName.DEFAULT) + .routingTable(routingTable) + .metadata(Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(1L).build())) + .version(2L) + .build(); } private BlobPath getPath() { diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java index f6999cf71c9e6..d5d1870e08d18 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java @@ -13,7 +13,7 @@ import org.opensearch.index.remote.RemoteStoreEnums.DataCategory; import org.opensearch.index.remote.RemoteStoreEnums.DataType; import org.opensearch.index.remote.RemoteStoreEnums.PathType; -import org.opensearch.index.remote.RemoteStorePathStrategy.RemoteStorePathInput; +import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; @@ -70,7 +70,7 @@ public void testGeneratePathForFixedType() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId + SEPARATOR; // Translog Data - RemoteStorePathInput pathInput = RemoteStorePathInput.builder() + PathInput pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -82,7 +82,7 @@ public void testGeneratePathForFixedType() { // Translog Metadata dataType = METADATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -92,21 +92,10 @@ public void testGeneratePathForFixedType() { result = FIXED.path(pathInput, null); assertEquals(basePath + dataCategory.getName() + SEPARATOR + dataType.getName() + SEPARATOR, result.buildAsString()); - // Translog Lock files - This is a negative case where the assertion will trip. - dataType = LOCK_FILES; - RemoteStorePathInput finalPathInput = RemoteStorePathInput.builder() - .basePath(blobPath) - .indexUUID(indexUUID) - .shardId(shardId) - .dataCategory(dataCategory) - .dataType(dataType) - .build(); - assertThrows(AssertionError.class, () -> FIXED.path(finalPathInput, null)); - // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -118,7 +107,7 @@ public void testGeneratePathForFixedType() { // Segment Metadata dataType = METADATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -130,7 +119,7 @@ public void testGeneratePathForFixedType() { // Segment Metadata dataType = LOCK_FILES; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -155,7 +144,7 @@ public void testGeneratePathForHashedPrefixType() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId; // Translog Data - RemoteStorePathInput pathInput = RemoteStorePathInput.builder() + PathInput pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -172,7 +161,7 @@ public void testGeneratePathForHashedPrefixType() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -184,7 +173,7 @@ public void testGeneratePathForHashedPrefixType() { // Translog Metadata dataType = METADATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -198,7 +187,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -208,31 +197,10 @@ public void testGeneratePathForHashedPrefixType() { result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertEquals("oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString()); - // Translog Lock files - This is a negative case where the assertion will trip. - dataType = LOCK_FILES; - RemoteStorePathInput finalPathInput = RemoteStorePathInput.builder() - .basePath(blobPath) - .indexUUID(indexUUID) - .shardId(shardId) - .dataCategory(dataCategory) - .dataType(dataType) - .build(); - assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null)); - - // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() - .basePath(fixedBlobPath) - .indexUUID(fixedIndexUUID) - .shardId(fixedShardId) - .dataCategory(dataCategory) - .dataType(dataType) - .build(); - assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null)); - // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -246,7 +214,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -258,7 +226,7 @@ public void testGeneratePathForHashedPrefixType() { // Segment Metadata dataType = METADATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -272,7 +240,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -284,7 +252,7 @@ public void testGeneratePathForHashedPrefixType() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -298,7 +266,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -323,7 +291,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId; // Translog Data - RemoteStorePathInput pathInput = RemoteStorePathInput.builder() + PathInput pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -342,7 +310,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -354,7 +322,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Translog Metadata dataType = METADATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -370,7 +338,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -383,31 +351,10 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { result.buildAsString() ); - // Translog Lock files - This is a negative case where the assertion will trip. - dataType = LOCK_FILES; - RemoteStorePathInput finalPathInput = RemoteStorePathInput.builder() - .basePath(blobPath) - .indexUUID(indexUUID) - .shardId(shardId) - .dataCategory(dataCategory) - .dataType(dataType) - .build(); - assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null)); - - // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() - .basePath(fixedBlobPath) - .indexUUID(fixedIndexUUID) - .shardId(fixedShardId) - .dataCategory(dataCategory) - .dataType(dataType) - .build(); - assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null)); - // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -423,7 +370,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -435,7 +382,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Segment Metadata dataType = METADATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -451,7 +398,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -466,7 +413,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -482,7 +429,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -511,7 +458,7 @@ public void testGeneratePathForHashedInfixType() { String basePath = getPath(pathList); basePath = basePath.length() == 0 ? basePath : basePath.substring(0, basePath.length() - 1); // Translog Data - RemoteStorePathInput pathInput = RemoteStorePathInput.builder() + PathInput pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -527,7 +474,7 @@ public void testGeneratePathForHashedInfixType() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -541,7 +488,7 @@ public void testGeneratePathForHashedInfixType() { // Translog Metadata dataType = METADATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -555,7 +502,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -567,31 +514,10 @@ public void testGeneratePathForHashedInfixType() { actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); - // Translog Lock files - This is a negative case where the assertion will trip. - dataType = LOCK_FILES; - RemoteStorePathInput finalPathInput = RemoteStorePathInput.builder() - .basePath(blobPath) - .indexUUID(indexUUID) - .shardId(shardId) - .dataCategory(dataCategory) - .dataType(dataType) - .build(); - assertThrows(AssertionError.class, () -> HASHED_INFIX.path(finalPathInput, null)); - - // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() - .basePath(fixedBlobPath) - .indexUUID(fixedIndexUUID) - .shardId(fixedShardId) - .dataCategory(dataCategory) - .dataType(dataType) - .build(); - assertThrows(AssertionError.class, () -> HASHED_INFIX.path(finalPathInput, null)); - // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -604,7 +530,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -618,7 +544,7 @@ public void testGeneratePathForHashedInfixType() { // Segment Metadata dataType = METADATA; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -631,7 +557,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -645,7 +571,7 @@ public void testGeneratePathForHashedInfixType() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -658,7 +584,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = RemoteStorePathInput.builder() + pathInput = PathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -671,7 +597,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); } - private String derivePath(String basePath, RemoteStorePathInput pathInput) { + private String derivePath(String basePath, PathInput pathInput) { return "".equals(basePath) ? String.join( SEPARATOR, diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java new file mode 100644 index 0000000000000..c17e6bf51014b --- /dev/null +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java @@ -0,0 +1,87 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.remote; + +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.test.OpenSearchTestCase; + +import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; +import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; +import static org.opensearch.index.remote.RemoteStoreEnums.DataType.LOCK_FILES; + +public class RemoteStorePathStrategyTests extends OpenSearchTestCase { + + private static final BlobPath BASE_PATH = BlobPath.cleanPath().add("base-path"); + private static final String INDEX_UUID = "indexUUID"; + private static final String SHARD_ID = "shardId"; + + public void testBasePathInput() { + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.BasePathInput.builder().build()); + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.BasePathInput.builder().basePath(BASE_PATH).build()); + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.BasePathInput.builder().indexUUID(INDEX_UUID).build()); + RemoteStorePathStrategy.BasePathInput input = RemoteStorePathStrategy.BasePathInput.builder() + .basePath(BASE_PATH) + .indexUUID(INDEX_UUID) + .build(); + assertEquals(BASE_PATH, input.basePath()); + assertEquals(INDEX_UUID, input.indexUUID()); + } + + public void testPathInput() { + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.PathInput.builder().build()); + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.PathInput.builder().shardId(SHARD_ID).build()); + assertThrows( + NullPointerException.class, + () -> RemoteStorePathStrategy.PathInput.builder().shardId(SHARD_ID).dataCategory(TRANSLOG).build() + ); + + // Translog Lock files - This is a negative case where the assertion will trip. + assertThrows( + AssertionError.class, + () -> RemoteStorePathStrategy.PathInput.builder() + .basePath(BASE_PATH) + .indexUUID(INDEX_UUID) + .shardId(SHARD_ID) + .dataCategory(TRANSLOG) + .dataType(LOCK_FILES) + .build() + ); + + RemoteStorePathStrategy.PathInput input = RemoteStorePathStrategy.PathInput.builder() + .basePath(BASE_PATH) + .indexUUID(INDEX_UUID) + .shardId(SHARD_ID) + .dataCategory(TRANSLOG) + .dataType(DATA) + .build(); + assertEquals(BASE_PATH, input.basePath()); + assertEquals(INDEX_UUID, input.indexUUID()); + assertEquals(SHARD_ID, input.shardId()); + assertEquals(DATA, input.dataType()); + assertEquals(TRANSLOG, input.dataCategory()); + } + + public void testVariablePath() { + RemoteStorePathStrategy.BasePathInput input = RemoteStorePathStrategy.BasePathInput.builder() + .basePath(BASE_PATH) + .indexUUID(INDEX_UUID) + .build(); + assertEquals(BlobPath.cleanPath().add(INDEX_UUID), input.variablePath()); + + RemoteStorePathStrategy.PathInput input2 = RemoteStorePathStrategy.PathInput.builder() + .basePath(BASE_PATH) + .indexUUID(INDEX_UUID) + .shardId(SHARD_ID) + .dataCategory(TRANSLOG) + .dataType(DATA) + .build(); + assertEquals(BlobPath.cleanPath().add(INDEX_UUID).add(SHARD_ID).add(TRANSLOG.getName()).add(DATA.getName()), input.variablePath()); + + } +} diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index fe062f016c467..5ee65e7ea1a1c 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -1828,7 +1828,7 @@ public static BlobPath getShardLevelBlobPath( ? RemoteStoreEnums.PathHashAlgorithm.valueOf(remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME)) : null : null; - RemoteStorePathStrategy.RemoteStorePathInput pathInput = RemoteStorePathStrategy.RemoteStorePathInput.builder() + RemoteStorePathStrategy.PathInput pathInput = RemoteStorePathStrategy.PathInput.builder() .basePath(basePath) .indexUUID(indexUUID) .shardId(shardId) From cfd28d9b043f4178df4243cc73888fd7ce942e97 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Sun, 9 Jun 2024 18:18:57 +0530 Subject: [PATCH 43/50] Recovering Refactor the path type and hash algorith code Signed-off-by: Himshikha Gupta --- .../remote/RemoteRoutingTableService.java | 3 +- .../index/remote/RemoteIndexPath.java | 3 +- .../index/remote/RemoteStoreEnums.java | 33 +++---- .../index/remote/RemoteStorePathStrategy.java | 86 ++++++++++++------- .../RemoteSegmentStoreDirectoryFactory.java | 4 +- .../RemoteStoreLockManagerFactory.java | 2 +- .../index/translog/RemoteFsTranslog.java | 4 +- .../RemoteRoutingTableServiceTests.java | 2 +- .../index/remote/RemoteStoreEnumsTests.java | 76 ++++++++-------- .../remote/RemoteStorePathStrategyTests.java | 26 +++--- .../opensearch/test/OpenSearchTestCase.java | 2 +- 11 files changed, 135 insertions(+), 106 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index d5d55f841df16..6530646cc019b 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -45,6 +45,7 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import java.io.IOException; import java.util.ArrayList; @@ -170,7 +171,7 @@ public CheckedRunnable getIndexRoutingAsyncAction( BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN); BlobPath path = pathType.path( - RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(), + PathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(), pathHashAlgo ); final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java index 89b642b79df86..899ff16c9d607 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java @@ -19,6 +19,7 @@ import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; +import org.opensearch.index.remote.RemoteStorePathStrategy.ShardDataPathInput; import java.io.IOException; import java.util.Collections; @@ -141,7 +142,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws DataCategory dataCategory = entry.getKey(); for (DataType type : entry.getValue()) { for (int shardNo = 0; shardNo < shardCount; shardNo++) { - PathInput pathInput = PathInput.builder() + PathInput pathInput = ShardDataPathInput.builder() .basePath(new BlobPath().add(basePath)) .indexUUID(indexUUID) .shardId(Integer.toString(shardNo)) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java index ba33cef142094..17fb241846a72 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java @@ -19,6 +19,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import static java.util.Collections.unmodifiableMap; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; @@ -91,10 +92,10 @@ public String getName() { public enum PathType { FIXED(0) { @Override - public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { + public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.isNull(hashAlgorithm) : "hashAlgorithm is expected to be null with fixed remote store path type"; // Hash algorithm is not used in FIXED path type - return pathInput.basePath().add(pathInput.variablePath()); + return pathInput.basePath().add(pathInput.fixedSubPath()); } @Override @@ -104,9 +105,9 @@ boolean requiresHashAlgorithm() { }, HASHED_PREFIX(1) { @Override - public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { + public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - return BlobPath.cleanPath().add(hashAlgorithm.hash(pathInput)).add(pathInput.basePath()).add(pathInput.variablePath()); + return BlobPath.cleanPath().add(hashAlgorithm.hash(pathInput)).add(pathInput.basePath()).add(pathInput.fixedSubPath()); } @Override @@ -116,9 +117,9 @@ boolean requiresHashAlgorithm() { }, HASHED_INFIX(2) { @Override - public BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { + public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - return pathInput.basePath().add(hashAlgorithm.hash(pathInput)).add(pathInput.variablePath()); + return pathInput.basePath().add(hashAlgorithm.hash(pathInput)).add(pathInput.fixedSubPath()); } @Override @@ -169,12 +170,12 @@ public static PathType fromCode(int code) { * @param hashAlgorithm hashing algorithm. * @return the blob path for the path input. */ - public BlobPath path(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) { - + public BlobPath path(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { + pathInput.assertIsValid(); return generatePath(pathInput, hashAlgorithm); } - protected abstract BlobPath generatePath(RemoteStorePathStrategy.BasePathInput pathInput, PathHashAlgorithm hashAlgorithm); + protected abstract BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm); abstract boolean requiresHashAlgorithm(); @@ -204,10 +205,10 @@ public enum PathHashAlgorithm { FNV_1A_BASE64(0) { @Override - String hash(RemoteStorePathStrategy.BasePathInput pathInput) { + String hash(PathInput pathInput) { StringBuilder input = new StringBuilder(); - for (String paths : pathInput.variablePath().toArray()) { - input.append(paths); + for (String path : pathInput.fixedSubPath().toArray()) { + input.append(path); } long hash = FNV1a.hash64(input.toString()); return longToUrlBase64(hash); @@ -219,10 +220,10 @@ String hash(RemoteStorePathStrategy.BasePathInput pathInput) { */ FNV_1A_COMPOSITE_1(1) { @Override - String hash(RemoteStorePathStrategy.BasePathInput pathInput) { + String hash(PathInput pathInput) { StringBuilder input = new StringBuilder(); - for (String paths : pathInput.variablePath().toArray()) { - input.append(paths); + for (String path : pathInput.fixedSubPath().toArray()) { + input.append(path); } long hash = FNV1a.hash64(input.toString()); return longToCompositeBase64AndBinaryEncoding(hash, 20); @@ -263,7 +264,7 @@ public static PathHashAlgorithm fromCode(int code) { return CODE_TO_ENUM.get(code); } - abstract String hash(RemoteStorePathStrategy.BasePathInput pathInput); + abstract String hash(PathInput pathInput); public static PathHashAlgorithm parseString(String pathHashAlgorithm) { try { diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java index 6907e936efad2..6dabf08a6fcf5 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java @@ -72,18 +72,20 @@ public BlobPath generatePath(PathInput pathInput) { } /** - * Wrapper class for the input required to generate path for remote store uploads. + * Wrapper class for the path input required to generate path for remote store uploads. This input is composed of + * basePath and indexUUID. + * * @opensearch.internal */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class BasePathInput { + public static class PathInput { private final BlobPath basePath; private final String indexUUID; - public BasePathInput(BlobPath basePath, String indexUUID) { - this.basePath = Objects.requireNonNull(basePath); - this.indexUUID = Objects.requireNonNull(indexUUID); + public PathInput(Builder builder) { + this.basePath = Objects.requireNonNull(builder.basePath); + this.indexUUID = Objects.requireNonNull(builder.indexUUID); } BlobPath basePath() { @@ -94,60 +96,70 @@ String indexUUID() { return indexUUID; } - BlobPath variablePath() { + BlobPath fixedSubPath() { return BlobPath.cleanPath().add(indexUUID); } /** - * Returns a new builder for {@link BasePathInput}. + * Returns a new builder for {@link PathInput}. */ - public static Builder builder() { - return new Builder(); + public static Builder builder() { + return new Builder<>(); + } + + public void assertIsValid() { + // Input is always valid here. } /** - * Builder for {@link BasePathInput}. + * Builder for {@link PathInput}. * * @opensearch.internal */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class Builder { + public static class Builder> { private BlobPath basePath; private String indexUUID; - public Builder basePath(BlobPath basePath) { + public T basePath(BlobPath basePath) { this.basePath = basePath; - return this; + return self(); } public Builder indexUUID(String indexUUID) { this.indexUUID = indexUUID; - return this; + return self(); + } + + protected T self() { + return (T) this; } - public BasePathInput build() { - return new BasePathInput(basePath, indexUUID); + public PathInput build() { + return new PathInput(this); } } } /** - * Wrapper class for the input required to generate path for remote store uploads. + * Wrapper class for the data aware path input required to generate path for remote store uploads. This input is + * composed of the parent inputs, shard id, data category and data type. + * * @opensearch.internal */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class PathInput extends BasePathInput { + public static class ShardDataPathInput extends PathInput { private final String shardId; private final DataCategory dataCategory; private final DataType dataType; - public PathInput(BlobPath basePath, String indexUUID, String shardId, DataCategory dataCategory, DataType dataType) { - super(basePath, indexUUID); - this.shardId = Objects.requireNonNull(shardId); - this.dataCategory = Objects.requireNonNull(dataCategory); - this.dataType = Objects.requireNonNull(dataType); + public ShardDataPathInput(Builder builder) { + super(builder); + this.shardId = Objects.requireNonNull(builder.shardId); + this.dataCategory = Objects.requireNonNull(builder.dataCategory); + this.dataType = Objects.requireNonNull(builder.dataType); assert dataCategory.isSupportedDataType(dataType) : "category:" + dataCategory + " type:" @@ -169,25 +181,34 @@ DataType dataType() { } @Override - BlobPath variablePath() { - return super.variablePath().add(shardId).add(dataCategory.getName()).add(dataType.getName()); + BlobPath fixedSubPath() { + return super.fixedSubPath().add(shardId).add(dataCategory.getName()).add(dataType.getName()); } + @Override + public void assertIsValid() { + super.assertIsValid(); + assert dataCategory.isSupportedDataType(dataType) : "category:" + + dataCategory + + " type:" + + dataType + + " are not supported together"; + } /** - * Returns a new builder for {@link PathInput}. + * Returns a new builder for {@link ShardDataPathInput}. */ public static Builder builder() { return new Builder(); } /** - * Builder for {@link PathInput}. + * Builder for {@link ShardDataPathInput}. * * @opensearch.internal */ @PublicApi(since = "2.14.0") @ExperimentalApi - public static class Builder extends BasePathInput.Builder { + public static class Builder extends PathInput.Builder { private String shardId; private DataCategory dataCategory; private DataType dataType; @@ -217,8 +238,13 @@ public Builder dataType(DataType dataType) { return this; } - public PathInput build() { - return new PathInput(super.basePath, super.indexUUID, shardId, dataCategory, dataType); + @Override + protected Builder self() { + return this; + } + + public ShardDataPathInput build() { + return new ShardDataPathInput(this); } } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index e462f6d4ac011..b965d7ce73ae6 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -65,7 +65,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s BlobPath repositoryBasePath = blobStoreRepository.basePath(); String shardIdStr = String.valueOf(shardId.id()); - RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.ShardDataPathInput dataPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardIdStr) @@ -80,7 +80,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s blobStoreRepository::maybeRateLimitRemoteDownloadTransfers ); - RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.ShardDataPathInput mdPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardIdStr) diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java index 45d466d3a8ce8..993c1bbdf033f 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java @@ -56,7 +56,7 @@ public static RemoteStoreMetadataLockManager newLockManager( assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobPath repositoryBasePath = ((BlobStoreRepository) repository).basePath(); - RemoteStorePathStrategy.PathInput lockFilesPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.ShardDataPathInput lockFilesPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(repositoryBasePath) .indexUUID(indexUUID) .shardId(shardId) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 67549c86b7dd2..50b9a5da0e81a 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -304,7 +304,7 @@ public static TranslogTransferManager buildTranslogTransferManager( assert Objects.nonNull(pathStrategy); String indexUUID = shardId.getIndex().getUUID(); String shardIdStr = String.valueOf(shardId.id()); - RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.ShardDataPathInput dataPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(blobStoreRepository.basePath()) .indexUUID(indexUUID) .shardId(shardIdStr) @@ -312,7 +312,7 @@ public static TranslogTransferManager buildTranslogTransferManager( .dataType(DATA) .build(); BlobPath dataPath = pathStrategy.generatePath(dataPathInput); - RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.ShardDataPathInput mdPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(blobStoreRepository.basePath()) .indexUUID(indexUUID) .shardId(shardIdStr) diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 467e6cfa6941d..7385d117a8f1e 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -548,7 +548,7 @@ private ClusterState createClusterState(String indexName) { private BlobPath getPath() { BlobPath indexRoutingPath = basePath.add(INDEX_ROUTING_PATH_TOKEN); return RemoteStoreEnums.PathType.HASHED_PREFIX.path( - RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID("uuid").build(), + RemoteStorePathStrategy.PathInput.builder().basePath(indexRoutingPath).indexUUID("uuid").build(), RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64 ); } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java index d5d1870e08d18..481a0568ff0a7 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java @@ -13,7 +13,7 @@ import org.opensearch.index.remote.RemoteStoreEnums.DataCategory; import org.opensearch.index.remote.RemoteStoreEnums.DataType; import org.opensearch.index.remote.RemoteStoreEnums.PathType; -import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; +import org.opensearch.index.remote.RemoteStorePathStrategy.ShardDataPathInput; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; @@ -70,7 +70,7 @@ public void testGeneratePathForFixedType() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId + SEPARATOR; // Translog Data - PathInput pathInput = PathInput.builder() + ShardDataPathInput pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -82,7 +82,7 @@ public void testGeneratePathForFixedType() { // Translog Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -95,7 +95,7 @@ public void testGeneratePathForFixedType() { // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -107,7 +107,7 @@ public void testGeneratePathForFixedType() { // Segment Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -119,7 +119,7 @@ public void testGeneratePathForFixedType() { // Segment Metadata dataType = LOCK_FILES; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -144,7 +144,7 @@ public void testGeneratePathForHashedPrefixType() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId; // Translog Data - PathInput pathInput = PathInput.builder() + ShardDataPathInput pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -161,7 +161,7 @@ public void testGeneratePathForHashedPrefixType() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -173,7 +173,7 @@ public void testGeneratePathForHashedPrefixType() { // Translog Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -187,7 +187,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -200,7 +200,7 @@ public void testGeneratePathForHashedPrefixType() { // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -214,7 +214,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -226,7 +226,7 @@ public void testGeneratePathForHashedPrefixType() { // Segment Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -240,7 +240,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -252,7 +252,7 @@ public void testGeneratePathForHashedPrefixType() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -266,7 +266,7 @@ public void testGeneratePathForHashedPrefixType() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -291,7 +291,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId; // Translog Data - PathInput pathInput = PathInput.builder() + ShardDataPathInput pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -310,7 +310,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -322,7 +322,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Translog Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -338,7 +338,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -354,7 +354,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -370,7 +370,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -382,7 +382,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Segment Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -398,7 +398,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -413,7 +413,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -429,7 +429,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { ); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -456,9 +456,9 @@ public void testGeneratePathForHashedInfixType() { DataType dataType = DATA; String basePath = getPath(pathList); - basePath = basePath.length() == 0 ? basePath : basePath.substring(0, basePath.length() - 1); + basePath = basePath.isEmpty() ? basePath : basePath.substring(0, basePath.length() - 1); // Translog Data - PathInput pathInput = PathInput.builder() + ShardDataPathInput pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -474,7 +474,7 @@ public void testGeneratePathForHashedInfixType() { BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String fixedIndexUUID = "k2ijhe877d7yuhx7"; String fixedShardId = "10"; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -488,7 +488,7 @@ public void testGeneratePathForHashedInfixType() { // Translog Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -502,7 +502,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -517,7 +517,7 @@ public void testGeneratePathForHashedInfixType() { // Segment Data dataCategory = SEGMENTS; dataType = DATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -530,7 +530,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -544,7 +544,7 @@ public void testGeneratePathForHashedInfixType() { // Segment Metadata dataType = METADATA; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -557,7 +557,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -571,7 +571,7 @@ public void testGeneratePathForHashedInfixType() { // Segment Lockfiles dataType = LOCK_FILES; - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) @@ -584,7 +584,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); // assert with exact value for known base path - pathInput = PathInput.builder() + pathInput = ShardDataPathInput.builder() .basePath(fixedBlobPath) .indexUUID(fixedIndexUUID) .shardId(fixedShardId) @@ -597,7 +597,7 @@ public void testGeneratePathForHashedInfixType() { assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); } - private String derivePath(String basePath, PathInput pathInput) { + private String derivePath(String basePath, ShardDataPathInput pathInput) { return "".equals(basePath) ? String.join( SEPARATOR, diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java index c17e6bf51014b..cfa97fc3b2251 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java @@ -22,10 +22,10 @@ public class RemoteStorePathStrategyTests extends OpenSearchTestCase { private static final String SHARD_ID = "shardId"; public void testBasePathInput() { - assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.BasePathInput.builder().build()); - assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.BasePathInput.builder().basePath(BASE_PATH).build()); - assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.BasePathInput.builder().indexUUID(INDEX_UUID).build()); - RemoteStorePathStrategy.BasePathInput input = RemoteStorePathStrategy.BasePathInput.builder() + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.PathInput.builder().build()); + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.PathInput.builder().basePath(BASE_PATH).build()); + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.PathInput.builder().indexUUID(INDEX_UUID).build()); + RemoteStorePathStrategy.PathInput input = RemoteStorePathStrategy.PathInput.builder() .basePath(BASE_PATH) .indexUUID(INDEX_UUID) .build(); @@ -34,17 +34,17 @@ public void testBasePathInput() { } public void testPathInput() { - assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.PathInput.builder().build()); - assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.PathInput.builder().shardId(SHARD_ID).build()); + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.ShardDataPathInput.builder().build()); + assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.ShardDataPathInput.builder().shardId(SHARD_ID).build()); assertThrows( NullPointerException.class, - () -> RemoteStorePathStrategy.PathInput.builder().shardId(SHARD_ID).dataCategory(TRANSLOG).build() + () -> RemoteStorePathStrategy.ShardDataPathInput.builder().shardId(SHARD_ID).dataCategory(TRANSLOG).build() ); // Translog Lock files - This is a negative case where the assertion will trip. assertThrows( AssertionError.class, - () -> RemoteStorePathStrategy.PathInput.builder() + () -> RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(BASE_PATH) .indexUUID(INDEX_UUID) .shardId(SHARD_ID) @@ -53,7 +53,7 @@ public void testPathInput() { .build() ); - RemoteStorePathStrategy.PathInput input = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.ShardDataPathInput input = RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(BASE_PATH) .indexUUID(INDEX_UUID) .shardId(SHARD_ID) @@ -68,20 +68,20 @@ public void testPathInput() { } public void testVariablePath() { - RemoteStorePathStrategy.BasePathInput input = RemoteStorePathStrategy.BasePathInput.builder() + RemoteStorePathStrategy.PathInput input = RemoteStorePathStrategy.PathInput.builder() .basePath(BASE_PATH) .indexUUID(INDEX_UUID) .build(); - assertEquals(BlobPath.cleanPath().add(INDEX_UUID), input.variablePath()); + assertEquals(BlobPath.cleanPath().add(INDEX_UUID), input.fixedSubPath()); - RemoteStorePathStrategy.PathInput input2 = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.ShardDataPathInput input2 = RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(BASE_PATH) .indexUUID(INDEX_UUID) .shardId(SHARD_ID) .dataCategory(TRANSLOG) .dataType(DATA) .build(); - assertEquals(BlobPath.cleanPath().add(INDEX_UUID).add(SHARD_ID).add(TRANSLOG.getName()).add(DATA.getName()), input.variablePath()); + assertEquals(BlobPath.cleanPath().add(INDEX_UUID).add(SHARD_ID).add(TRANSLOG.getName()).add(DATA.getName()), input.fixedSubPath()); } } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 5ee65e7ea1a1c..6afc7c23d9e66 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -1828,7 +1828,7 @@ public static BlobPath getShardLevelBlobPath( ? RemoteStoreEnums.PathHashAlgorithm.valueOf(remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME)) : null : null; - RemoteStorePathStrategy.PathInput pathInput = RemoteStorePathStrategy.PathInput.builder() + RemoteStorePathStrategy.ShardDataPathInput pathInput = RemoteStorePathStrategy.ShardDataPathInput.builder() .basePath(basePath) .indexUUID(indexUUID) .shardId(shardId) From e6f41f65ba8b9dae9351d25c52285fbd1b3823fe Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Sun, 9 Jun 2024 17:30:17 +0530 Subject: [PATCH 44/50] moving remoteRoutingTableService init to factory Signed-off-by: Himshikha Gupta --- .../DefaultRemoteRoutingTableService.java | 303 ++++++++++++++++++ .../remote/NoopRemoteRoutingTableService.java | 74 +++++ .../remote/RemoteRoutingTableService.java | 270 +--------------- .../RemoteRoutingTableServiceFactory.java | 41 +++ .../common/settings/ClusterSettings.java | 6 +- .../remote/RemoteClusterStateService.java | 66 ++-- ...RemoteRoutingTableServiceFactoryTests.java | 51 +++ .../RemoteRoutingTableServiceTests.java | 40 +-- .../remote/ClusterMetadataManifestTests.java | 4 +- .../RemoteClusterStateServiceTests.java | 13 +- 10 files changed, 539 insertions(+), 329 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/routing/remote/DefaultRemoteRoutingTableService.java create mode 100644 server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java create mode 100644 server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java create mode 100644 server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/DefaultRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/DefaultRemoteRoutingTableService.java new file mode 100644 index 0000000000000..f929c670f768c --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/DefaultRemoteRoutingTableService.java @@ -0,0 +1,303 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.remote; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.IndexInput; +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; +import org.opensearch.common.lucene.store.ByteArrayIndexInput; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.index.remote.RemoteStorePathStrategy; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.blobstore.BlobStoreRepository; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; + +/** + * A Service which provides APIs to upload and download routing table from remote store. + * + * @opensearch.internal + */ +public class DefaultRemoteRoutingTableService extends AbstractLifecycleComponent implements RemoteRoutingTableService { + + /** + * This setting is used to set the remote routing table store blob store path type strategy. + */ + public static final Setting REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING = new Setting<>( + "cluster.remote_store.routing_table.path_type", + RemoteStoreEnums.PathType.HASHED_PREFIX.toString(), + RemoteStoreEnums.PathType::parseString, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting is used to set the remote routing table store blob store path hash algorithm strategy. + * This setting will come to effect if the {@link #REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING} + * is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}. + */ + public static final Setting REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING = new Setting<>( + "cluster.remote_store.routing_table.path_hash_algo", + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64.toString(), + RemoteStoreEnums.PathHashAlgorithm::parseString, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final String INDEX_ROUTING_PATH_TOKEN = "index-routing"; + public static final String INDEX_ROUTING_FILE_PREFIX = "index_routing"; + public static final String DELIMITER = "__"; + public static final String INDEX_ROUTING_METADATA_PREFIX = "indexRouting--"; + + private static final Logger logger = LogManager.getLogger(DefaultRemoteRoutingTableService.class); + private final Settings settings; + private final Supplier repositoriesService; + private BlobStoreRepository blobStoreRepository; + private RemoteStoreEnums.PathType pathType; + private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo; + + public DefaultRemoteRoutingTableService( + Supplier repositoriesService, + Settings settings, + ClusterSettings clusterSettings + ) { + assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; + this.repositoriesService = repositoriesService; + this.settings = settings; + this.pathType = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING); + this.pathHashAlgo = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING); + clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, this::setPathTypeSetting); + clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, this::setPathHashAlgoSetting); + } + + private void setPathTypeSetting(RemoteStoreEnums.PathType pathType) { + this.pathType = pathType; + } + + private void setPathHashAlgoSetting(RemoteStoreEnums.PathHashAlgorithm pathHashAlgo) { + this.pathHashAlgo = pathHashAlgo; + } + + public List getIndicesRouting(RoutingTable routingTable) { + return new ArrayList<>(routingTable.indicesRouting().values()); + } + + /** + * Returns diff between the two routing tables, which includes upserts and deletes. + * @param before previous routing table + * @param after current routing table + * @return diff of the previous and current routing table + */ + public DiffableUtils.MapDiff> getIndicesRoutingMapDiff( + RoutingTable before, + RoutingTable after + ) { + return DiffableUtils.diff( + before.getIndicesRouting(), + after.getIndicesRouting(), + DiffableUtils.getStringKeySerializer(), + CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER + ); + } + + /** + * Create async action for writing one {@code IndexRoutingTable} to remote store + * @param clusterState current cluster state + * @param indexRouting indexRoutingTable to write to remote store + * @param latchedActionListener listener for handling async action response + * @param clusterBasePath base path for remote file + * @return returns runnable async action + */ + public CheckedRunnable getIndexRoutingAsyncAction( + ClusterState clusterState, + IndexRoutingTable indexRouting, + LatchedActionListener latchedActionListener, + BlobPath clusterBasePath + ) { + + BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN); + BlobPath path = pathType.path( + RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(), + pathHashAlgo + ); + final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); + + final String fileName = getIndexRoutingFileName(clusterState.term(), clusterState.version()); + + ActionListener completionListener = ActionListener.wrap( + resp -> latchedActionListener.onResponse( + new ClusterMetadataManifest.UploadedIndexMetadata( + indexRouting.getIndex().getName(), + indexRouting.getIndex().getUUID(), + path.buildAsString() + fileName, + INDEX_ROUTING_METADATA_PREFIX + ) + ), + ex -> latchedActionListener.onFailure( + new RemoteClusterStateService.RemoteStateTransferException( + "Exception in writing index to remote store: " + indexRouting.getIndex().toString(), + ex + ) + ) + ); + + return () -> uploadIndex(indexRouting, fileName, blobContainer, completionListener); + } + + /** + * Combines IndicesRoutingMetadata from previous manifest and current uploaded indices, removes deleted indices. + * @param previousManifest previous manifest, used to get all existing indices routing paths + * @param indicesRoutingUploaded current uploaded indices routings + * @param indicesRoutingToDelete indices to delete + * @return combined list of metadata + */ + public List getAllUploadedIndicesRouting( + ClusterMetadataManifest previousManifest, + List indicesRoutingUploaded, + List indicesRoutingToDelete + ) { + final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() + .stream() + .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); + + indicesRoutingUploaded.forEach( + uploadedIndexRouting -> allUploadedIndicesRouting.put(uploadedIndexRouting.getIndexName(), uploadedIndexRouting) + ); + indicesRoutingToDelete.forEach(allUploadedIndicesRouting::remove); + + return new ArrayList<>(allUploadedIndicesRouting.values()); + } + + private void uploadIndex( + IndexRoutingTable indexRouting, + String fileName, + BlobContainer blobContainer, + ActionListener completionListener + ) { + RemoteIndexRoutingTable indexRoutingInput = new RemoteIndexRoutingTable(indexRouting); + BytesReference bytesInput = null; + try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { + indexRoutingInput.writeTo(streamOutput); + bytesInput = streamOutput.bytes(); + } catch (IOException e) { + logger.error("Failed to serialize IndexRoutingTable for {}. ", indexRouting, e); + completionListener.onFailure(e); + return; + } + + if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { + try { + blobContainer.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); + completionListener.onResponse(null); + } catch (IOException e) { + logger.error("Failed to write IndexRoutingTable to remote store for indexRouting {}. ", indexRouting, e); + completionListener.onFailure(e); + } + return; + } + + try (IndexInput input = new ByteArrayIndexInput("indexrouting", BytesReference.toBytes(bytesInput))) { + try ( + RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( + fileName, + fileName, + input.length(), + true, + WritePriority.URGENT, + (size, position) -> new OffsetRangeIndexInputStream(input, size, position), + null, + false + ) + ) { + ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload( + remoteTransferContainer.createWriteContext(), + completionListener + ); + } catch (IOException e) { + logger.error("Failed to write IndexRoutingTable to remote store for indexRouting {}. ", indexRouting, e); + completionListener.onFailure(e); + } + } catch (IOException e) { + logger.error( + "Failed to create transfer object for IndexRoutingTable for remote store upload for indexRouting {}. ", + indexRouting, + e + ); + completionListener.onFailure(e); + } + } + + private String getIndexRoutingFileName(long term, long version) { + return String.join( + DELIMITER, + INDEX_ROUTING_FILE_PREFIX, + RemoteStoreUtils.invertLong(term), + RemoteStoreUtils.invertLong(version), + RemoteStoreUtils.invertLong(System.currentTimeMillis()) + ); + } + + @Override + protected void doClose() throws IOException { + if (blobStoreRepository != null) { + IOUtils.close(blobStoreRepository); + } + } + + @Override + protected void doStart() { + assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled"; + final String remoteStoreRepo = settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY + ); + assert remoteStoreRepo != null : "Remote routing table repository is not configured"; + final Repository repository = repositoriesService.get().repository(remoteStoreRepo); + assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; + blobStoreRepository = (BlobStoreRepository) repository; + } + + @Override + protected void doStop() {} + +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java new file mode 100644 index 0000000000000..99933c72195e9 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.remote; + +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; +import org.opensearch.gateway.remote.ClusterMetadataManifest; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class NoopRemoteRoutingTableService extends AbstractLifecycleComponent implements RemoteRoutingTableService { + + @Override + public List getIndicesRouting(RoutingTable routingTable) { + return List.of(); + } + + @Override + public DiffableUtils.MapDiff> getIndicesRoutingMapDiff( + RoutingTable before, + RoutingTable after + ) { + return DiffableUtils.diff(Map.of(), Map.of(), DiffableUtils.getStringKeySerializer(), CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER); + } + + @Override + public CheckedRunnable getIndexRoutingAsyncAction( + ClusterState clusterState, + IndexRoutingTable indexRouting, + LatchedActionListener latchedActionListener, + BlobPath clusterBasePath + ) { + //noop + return () -> {}; + } + + @Override + public List getAllUploadedIndicesRouting( + ClusterMetadataManifest previousManifest, + List indicesRoutingUploaded, + List indicesRoutingToDelete + ) { + return List.of(); + } + + @Override + protected void doStart() { + // noop + } + + @Override + protected void doStop() { + // noop + } + + @Override + protected void doClose() throws IOException { + // noop + } +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 6530646cc019b..dbf01904116ed 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -8,121 +8,27 @@ package org.opensearch.cluster.routing.remote; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.store.IndexInput; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; -import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.blobstore.stream.write.WritePriority; -import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; -import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.lifecycle.AbstractLifecycleComponent; -import org.opensearch.common.lucene.store.ByteArrayIndexInput; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.io.IOUtils; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.common.lifecycle.LifecycleComponent; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.gateway.remote.ClusterMetadataManifest; -import org.opensearch.gateway.remote.RemoteClusterStateService; -import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable; -import org.opensearch.index.remote.RemoteStoreEnums; -import org.opensearch.index.remote.RemoteStorePathStrategy; -import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.node.Node; -import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; -import org.opensearch.repositories.RepositoriesService; -import org.opensearch.repositories.Repository; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; /** - * A Service which provides APIs to upload and download routing table from remote store. - * - * @opensearch.internal + * Interface for RemoteRoutingTableService. Exposes methods to orchestrate upload and download of routing table from remote store. */ -public class RemoteRoutingTableService extends AbstractLifecycleComponent { - - /** - * This setting is used to set the remote routing table store blob store path type strategy. - */ - public static final Setting REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING = new Setting<>( - "cluster.remote_store.routing_table.path_type", - RemoteStoreEnums.PathType.HASHED_PREFIX.toString(), - RemoteStoreEnums.PathType::parseString, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting is used to set the remote routing table store blob store path hash algorithm strategy. - * This setting will come to effect if the {@link #REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING} - * is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}. - */ - public static final Setting REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING = new Setting<>( - "cluster.remote_store.routing_table.path_hash_algo", - RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64.toString(), - RemoteStoreEnums.PathHashAlgorithm::parseString, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - public static final String INDEX_ROUTING_PATH_TOKEN = "index-routing"; - public static final String INDEX_ROUTING_FILE_PREFIX = "index_routing"; - public static final String DELIMITER = "__"; - public static final String INDEX_ROUTING_METADATA_PREFIX = "indexRouting--"; - - private static final Logger logger = LogManager.getLogger(RemoteRoutingTableService.class); - private final Settings settings; - private final Supplier repositoriesService; - private BlobStoreRepository blobStoreRepository; - private RemoteStoreEnums.PathType pathType; - private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo; - - public RemoteRoutingTableService( - Supplier repositoriesService, - Settings settings, - ClusterSettings clusterSettings - ) { - assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; - this.repositoriesService = repositoriesService; - this.settings = settings; - this.pathType = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING); - this.pathHashAlgo = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING); - clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, this::setPathTypeSetting); - clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, this::setPathHashAlgoSetting); - } - - private void setPathTypeSetting(RemoteStoreEnums.PathType pathType) { - this.pathType = pathType; - } - - private void setPathHashAlgoSetting(RemoteStoreEnums.PathHashAlgorithm pathHashAlgo) { - this.pathHashAlgo = pathHashAlgo; - } - - private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = +public interface RemoteRoutingTableService extends LifecycleComponent { + static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = new DiffableUtils.NonDiffableValueSerializer() { @Override public void write(IndexRoutingTable value, StreamOutput out) throws IOException { @@ -135,176 +41,24 @@ public IndexRoutingTable read(StreamInput in, String key) throws IOException { } }; - /** - * Returns diff between the two routing tables, which includes upserts and deletes. - * @param before previous routing table - * @param after current routing table - * @return diff of the previous and current routing table - */ - public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff( + List getIndicesRouting(RoutingTable routingTable); + + DiffableUtils.MapDiff> getIndicesRoutingMapDiff( RoutingTable before, RoutingTable after - ) { - return DiffableUtils.diff( - before.getIndicesRouting(), - after.getIndicesRouting(), - DiffableUtils.getStringKeySerializer(), - CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER - ); - } + ); - /** - * Create async action for writing one {@code IndexRoutingTable} to remote store - * @param clusterState current cluster state - * @param indexRouting indexRoutingTable to write to remote store - * @param latchedActionListener listener for handling async action response - * @param clusterBasePath base path for remote file - * @return returns runnable async action - * @throws IOException exception thrown on failure in writing to remote store - */ - public CheckedRunnable getIndexRoutingAsyncAction( + CheckedRunnable getIndexRoutingAsyncAction( ClusterState clusterState, IndexRoutingTable indexRouting, LatchedActionListener latchedActionListener, BlobPath clusterBasePath - ) throws IOException { - - BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN); - BlobPath path = pathType.path( - PathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(), - pathHashAlgo - ); - final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); - - final String fileName = getIndexRoutingFileName(clusterState.term(), clusterState.version()); - - ActionListener completionListener = ActionListener.wrap( - resp -> latchedActionListener.onResponse( - new ClusterMetadataManifest.UploadedIndexMetadata( - indexRouting.getIndex().getName(), - indexRouting.getIndex().getUUID(), - path.buildAsString() + fileName, - INDEX_ROUTING_METADATA_PREFIX - ) - ), - ex -> latchedActionListener.onFailure( - new RemoteClusterStateService.RemoteStateTransferException( - "Exception in writing index to remote store: " + indexRouting.getIndex().toString(), - ex - ) - ) - ); - - return () -> uploadIndex(indexRouting, fileName, blobContainer, completionListener); - } + ); - /** - * Combines IndicesRoutingMetadata from previous manifest and current uploaded indices, removes deleted indices. - * @param previousManifest previous manifest, used to get all existing indices routing paths - * @param indicesRoutingUploaded current uploaded indices routings - * @param indicesRoutingToDelete indices to delete - * @return combined list of metadata - */ - public List getAllUploadedIndicesRouting( + List getAllUploadedIndicesRouting( ClusterMetadataManifest previousManifest, List indicesRoutingUploaded, List indicesRoutingToDelete - ) { - final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() - .stream() - .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); - - indicesRoutingUploaded.forEach( - uploadedIndexRouting -> allUploadedIndicesRouting.put(uploadedIndexRouting.getIndexName(), uploadedIndexRouting) - ); - indicesRoutingToDelete.forEach(allUploadedIndicesRouting::remove); - - return new ArrayList<>(allUploadedIndicesRouting.values()); - } - - private void uploadIndex( - IndexRoutingTable indexRouting, - String fileName, - BlobContainer blobContainer, - ActionListener completionListener - ) throws IOException { - RemoteIndexRoutingTable indexRoutingInput = new RemoteIndexRoutingTable(indexRouting); - BytesReference bytesInput = null; - try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { - indexRoutingInput.writeTo(streamOutput); - bytesInput = streamOutput.bytes(); - } catch (IOException e) { - throw new IOException("Failed to serialize IndexRoutingTable. ", e); - } - - if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { - try { - blobContainer.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); - completionListener.onResponse(null); - } catch (IOException e) { - logger.error("Failed to write IndexRoutingTable to remote store. ", e); - completionListener.onFailure(e); - } - return; - } - - try (IndexInput input = new ByteArrayIndexInput("indexrouting", BytesReference.toBytes(bytesInput))) { - try ( - RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( - fileName, - fileName, - input.length(), - true, - WritePriority.URGENT, - (size, position) -> new OffsetRangeIndexInputStream(input, size, position), - null, - false - ) - ) { - ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload( - remoteTransferContainer.createWriteContext(), - completionListener - ); - } catch (IOException e) { - logger.error("Failed to write IndexRoutingTable to remote store. ", e); - completionListener.onFailure(e); - } - } catch (IOException e) { - logger.error("Failed to create transfer object for IndexRoutingTable for remote store upload. ", e); - completionListener.onFailure(e); - } - } - - private String getIndexRoutingFileName(long term, long version) { - return String.join( - DELIMITER, - INDEX_ROUTING_FILE_PREFIX, - RemoteStoreUtils.invertLong(term), - RemoteStoreUtils.invertLong(version), - RemoteStoreUtils.invertLong(System.currentTimeMillis()) - ); - } - - @Override - protected void doClose() throws IOException { - if (blobStoreRepository != null) { - IOUtils.close(blobStoreRepository); - } - } - - @Override - protected void doStart() { - assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled"; - final String remoteStoreRepo = settings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY - ); - assert remoteStoreRepo != null : "Remote routing table repository is not configured"; - final Repository repository = repositoriesService.get().repository(remoteStoreRepo); - assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; - blobStoreRepository = (BlobStoreRepository) repository; - } - - @Override - protected void doStop() {} + ); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java new file mode 100644 index 0000000000000..41c441c635e5c --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.remote; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.repositories.RepositoriesService; + +import java.util.function.Supplier; + +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; + +/** + * Factory to provide impl for RemoteRoutingTableService based on settings. + */ +public class RemoteRoutingTableServiceFactory { + + /** + * Returns {@code DefaultRemoteRoutingTableService} if the feature is enabled, otherwise {@code NoopRemoteRoutingTableService} + * @param repositoriesService repositoriesService + * @param settings settings + * @param clusterSettings clusterSettings + * @return RemoteRoutingTableService + */ + public static RemoteRoutingTableService getService( + Supplier repositoriesService, + Settings settings, + ClusterSettings clusterSettings + ) { + if (isRemoteRoutingTableEnabled(settings)) { + return new DefaultRemoteRoutingTableService(repositoriesService, settings, clusterSettings); + } + return new NoopRemoteRoutingTableService(); + } +} diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 1eeb8142ff5c8..c0592728c89ff 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -77,7 +77,7 @@ import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterManagerService; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; @@ -722,8 +722,8 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING, - RemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, - RemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, + DefaultRemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, + DefaultRemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, // Admission Control Settings AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 9b9b438619b82..0b50ba7eb08b6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -20,7 +20,9 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableServiceFactory; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -72,7 +74,6 @@ import static java.util.Objects.requireNonNull; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** @@ -211,7 +212,7 @@ public class RemoteClusterStateService implements Closeable { private final List indexMetadataUploadListeners; private BlobStoreRepository blobStoreRepository; private BlobStoreTransferService blobStoreTransferService; - private final Optional remoteRoutingTableService; + private final RemoteRoutingTableService remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; private volatile TimeValue indexMetadataUploadTimeout; @@ -264,9 +265,7 @@ public RemoteClusterStateService( this.remoteStateStats = new RemotePersistenceStats(); this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService); this.indexMetadataUploadListeners = indexMetadataUploadListeners; - this.remoteRoutingTableService = isRemoteRoutingTableEnabled(settings) - ? Optional.of(new RemoteRoutingTableService(repositoriesService, settings, clusterSettings)) - : Optional.empty(); + this.remoteRoutingTableService = RemoteRoutingTableServiceFactory.getService(repositoriesService, settings, clusterSettings); } /** @@ -291,7 +290,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri true, true, true, - remoteRoutingTableService.isPresent() ? new ArrayList<>(clusterState.getRoutingTable().indicesRouting().values()) : List.of() + remoteRoutingTableService.getIndicesRouting(clusterState.getRoutingTable()) ); final ClusterMetadataManifest manifest = uploadManifest( clusterState, @@ -387,15 +386,10 @@ public ClusterMetadataManifest writeIncrementalMetadata( indicesToBeDeletedFromRemote.remove(indexMetadata.getIndex().getName()); } + DiffableUtils.MapDiff> routingTableDiff = remoteRoutingTableService + .getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable()); List indicesRoutingToUpload = new ArrayList<>(); - DiffableUtils.MapDiff> routingTableDiff = null; - if (remoteRoutingTableService.isPresent()) { - routingTableDiff = RemoteRoutingTableService.getIndicesRoutingMapDiff( - previousClusterState.getRoutingTable(), - clusterState.getRoutingTable() - ); - routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v)); - } + routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v)); UploadedMetadataResults uploadedMetadataResults; // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, @@ -430,14 +424,11 @@ public ClusterMetadataManifest writeIncrementalMetadata( indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove); List allUploadedIndicesRouting = new ArrayList<>(); - if (remoteRoutingTableService.isPresent()) { - allUploadedIndicesRouting = remoteRoutingTableService.get() - .getAllUploadedIndicesRouting( - previousManifest, - uploadedMetadataResults.uploadedIndicesRoutingMetadata, - routingTableDiff.getDeletes() - ); - } + allUploadedIndicesRouting = remoteRoutingTableService.getAllUploadedIndicesRouting( + previousManifest, + uploadedMetadataResults.uploadedIndicesRoutingMetadata, + routingTableDiff.getDeletes() + ); final ClusterMetadataManifest manifest = uploadManifest( clusterState, @@ -564,20 +555,15 @@ private UploadedMetadataResults writeMetadataInParallel( }); indicesRoutingToUpload.forEach(indexRoutingTable -> { - try { - uploadTasks.put( - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName(), - remoteRoutingTableService.get() - .getIndexRoutingAsyncAction( - clusterState, - indexRoutingTable, - listener, - getCusterMetadataBasePath(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()) - ) - ); - } catch (IOException e) { - throw new RemoteStateTransferException("Failed to create upload tasks for index routing table", e); - } + uploadTasks.put( + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName(), + remoteRoutingTableService.getIndexRoutingAsyncAction( + clusterState, + indexRoutingTable, + listener, + getCusterMetadataBasePath(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()) + ) + ); }); // start async upload of all required metadata files @@ -627,7 +613,7 @@ private UploadedMetadataResults writeMetadataInParallel( UploadedMetadataResults response = new UploadedMetadataResults(); results.forEach((name, uploadedMetadata) -> { if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class) - && uploadedMetadata.getComponent().contains(RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) { + && uploadedMetadata.getComponent().contains(DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) { response.uploadedIndicesRoutingMetadata.add((UploadedIndexMetadata) uploadedMetadata); } else if (name.contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom-- @@ -814,7 +800,7 @@ public void close() throws IOException { if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } - this.remoteRoutingTableService.ifPresent(RemoteRoutingTableService::close); + this.remoteRoutingTableService.close(); } public void start() { @@ -827,7 +813,7 @@ public void start() { assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; remoteClusterStateCleanupManager.start(); - this.remoteRoutingTableService.ifPresent(RemoteRoutingTableService::start); + this.remoteRoutingTableService.start(); } private ClusterMetadataManifest uploadManifest( @@ -1013,7 +999,7 @@ public TimeValue getMetadataManifestUploadTimeout() { } // Package private for unit test - Optional getRemoteRoutingTableService() { + RemoteRoutingTableService getRemoteRoutingTableService() { return this.remoteRoutingTableService; } diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java new file mode 100644 index 0000000000000..717610a57f8f0 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.remote; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.function.Supplier; + +import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; + +public class RemoteRoutingTableServiceFactoryTests extends OpenSearchTestCase { + + Supplier repositoriesService; + + public void testGetServiceWhenRemoteRoutingDisabled() { + Settings settings = Settings.builder().build(); + RemoteRoutingTableService service = RemoteRoutingTableServiceFactory.getService( + repositoriesService, + settings, + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + assertTrue(service instanceof NoopRemoteRoutingTableService); + } + + public void testGetServiceWhenRemoteRoutingEnabled() { + Settings settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .put(FsRepository.REPOSITORIES_COMPRESS_SETTING.getKey(), false) + .build(); + Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + RemoteRoutingTableService service = RemoteRoutingTableServiceFactory.getService( + repositoriesService, + settings, + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + assertTrue(service instanceof DefaultRemoteRoutingTableService); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 7385d117a8f1e..7c5c9dfc27a32 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -54,8 +54,8 @@ import org.mockito.ArgumentCaptor; -import static org.opensearch.cluster.routing.remote.RemoteRoutingTableService.INDEX_ROUTING_FILE_PREFIX; -import static org.opensearch.cluster.routing.remote.RemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN; +import static org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService.INDEX_ROUTING_FILE_PREFIX; +import static org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -73,7 +73,7 @@ public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { - private RemoteRoutingTableService remoteRoutingTableService; + private DefaultRemoteRoutingTableService remoteRoutingTableService; private Supplier repositoriesServiceSupplier; private RepositoriesService repositoriesService; private BlobStoreRepository blobStoreRepository; @@ -104,7 +104,7 @@ public void setup() { basePath = BlobPath.cleanPath().add("base-path"); - remoteRoutingTableService = new RemoteRoutingTableService( + remoteRoutingTableService = new DefaultRemoteRoutingTableService( repositoriesServiceSupplier, settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) @@ -121,7 +121,7 @@ public void testFailInitializationWhenRemoteRoutingDisabled() { final Settings settings = Settings.builder().build(); assertThrows( AssertionError.class, - () -> new RemoteRoutingTableService( + () -> new DefaultRemoteRoutingTableService( repositoriesServiceSupplier, settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) @@ -152,7 +152,7 @@ public void testGetIndicesRoutingMapDiff() { RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); - DiffableUtils.MapDiff> diff = RemoteRoutingTableService + DiffableUtils.MapDiff> diff = remoteRoutingTableService .getIndicesRoutingMapDiff(routingTable, routingTable); assertEquals(0, diff.getUpserts().size()); assertEquals(0, diff.getDeletes().size()); @@ -165,7 +165,7 @@ public void testGetIndicesRoutingMapDiff() { .build(); RoutingTable routingTable2 = RoutingTable.builder().add(indexRoutingTableReversed).build(); - diff = RemoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable2); + diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable2); assertEquals(0, diff.getUpserts().size()); assertEquals(0, diff.getDeletes().size()); } @@ -191,7 +191,7 @@ public void testGetIndicesRoutingMapDiffIndexAdded() { ).numberOfShards(noOfShards).numberOfReplicas(noOfReplicas).build(); RoutingTable routingTable2 = RoutingTable.builder(routingTable).addAsNew(indexMetadata2).build(); - DiffableUtils.MapDiff> diff = RemoteRoutingTableService + DiffableUtils.MapDiff> diff = remoteRoutingTableService .getIndicesRoutingMapDiff(routingTable, routingTable2); assertEquals(1, diff.getUpserts().size()); assertNotNull(diff.getUpserts().get(indexName2)); @@ -222,7 +222,7 @@ public void testGetIndicesRoutingMapDiffShardChanged() { ).numberOfShards(noOfShards + 1).numberOfReplicas(noOfReplicas).build(); RoutingTable routingTable2 = RoutingTable.builder().addAsNew(indexMetadata2).build(); - DiffableUtils.MapDiff> diff = RemoteRoutingTableService + DiffableUtils.MapDiff> diff = remoteRoutingTableService .getIndicesRoutingMapDiff(routingTable, routingTable2); assertEquals(1, diff.getUpserts().size()); assertNotNull(diff.getUpserts().get(indexName)); @@ -238,7 +238,7 @@ public void testGetIndicesRoutingMapDiffShardChanged() { ).numberOfShards(noOfShards + 1).numberOfReplicas(noOfReplicas + 1).build(); RoutingTable routingTable3 = RoutingTable.builder().addAsNew(indexMetadata3).build(); - diff = RemoteRoutingTableService.getIndicesRoutingMapDiff(routingTable2, routingTable3); + diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable2, routingTable3); assertEquals(1, diff.getUpserts().size()); assertNotNull(diff.getUpserts().get(indexName)); assertEquals(noOfShards + 1, diff.getUpserts().get(indexName).getShards().size()); @@ -262,7 +262,7 @@ public void testGetIndicesRoutingMapDiffShardDetailChanged() { RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); RoutingTable routingTable2 = RoutingTable.builder().addAsRecovery(indexMetadata).build(); - DiffableUtils.MapDiff> diff = RemoteRoutingTableService + DiffableUtils.MapDiff> diff = remoteRoutingTableService .getIndicesRoutingMapDiff(routingTable, routingTable2); assertEquals(1, diff.getUpserts().size()); assertNotNull(diff.getUpserts().get(indexName)); @@ -290,7 +290,7 @@ public void testGetIndicesRoutingMapDiffIndexDeleted() { ).numberOfShards(randomInt(1000)).numberOfReplicas(randomInt(10)).build(); RoutingTable routingTable2 = RoutingTable.builder().addAsNew(indexMetadata2).build(); - DiffableUtils.MapDiff> diff = RemoteRoutingTableService + DiffableUtils.MapDiff> diff = remoteRoutingTableService .getIndicesRoutingMapDiff(routingTable, routingTable2); assertEquals(1, diff.getUpserts().size()); assertNotNull(diff.getUpserts().get(indexName2)); @@ -428,7 +428,7 @@ public void testGetAllUploadedIndicesRouting() { "test-index", "index-uuid", "index-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); List allIndiceRoutingMetadata = remoteRoutingTableService @@ -443,7 +443,7 @@ public void testGetAllUploadedIndicesRoutingExistingIndexInManifest() { "test-index", "index-uuid", "index-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indicesRouting(List.of(uploadedIndexMetadata)) @@ -461,7 +461,7 @@ public void testGetAllUploadedIndicesRoutingNewIndexFromManifest() { "test-index", "index-uuid", "index-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indicesRouting(List.of(uploadedIndexMetadata)) @@ -470,7 +470,7 @@ public void testGetAllUploadedIndicesRoutingNewIndexFromManifest() { "test-index2", "index-uuid", "index-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); List allIndiceRoutingMetadata = remoteRoutingTableService @@ -486,13 +486,13 @@ public void testGetAllUploadedIndicesRoutingIndexDeleted() { "test-index", "index-uuid", "index-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata( "test-index2", "index-uuid", "index-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indicesRouting(List.of(uploadedIndexMetadata, uploadedIndexMetadata2)) @@ -510,13 +510,13 @@ public void testGetAllUploadedIndicesRoutingNoChange() { "test-index", "index-uuid", "index-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata( "test-index2", "index-uuid", "index-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indicesRouting(List.of(uploadedIndexMetadata, uploadedIndexMetadata2)) diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 1e911476a33e6..9705c284b862b 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -12,7 +12,7 @@ import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -371,7 +371,7 @@ public void testClusterMetadataManifestXContentV3() throws IOException { "test-index", "test-uuid", "routing-path", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 5efc89e816aa7..b96ba68142879 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -20,7 +20,8 @@ import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RoutingTable; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.NoopRemoteRoutingTableService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; @@ -1388,7 +1389,7 @@ public void testGlobalMetadataUploadWaitTimeSetting() { } public void testRemoteRoutingTableNotInitializedWhenDisabled() { - assertFalse(remoteClusterStateService.getRemoteRoutingTableService().isPresent()); + assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof NoopRemoteRoutingTableService); } public void testRemoteRoutingTableInitializedWhenEnabled() { @@ -1411,7 +1412,7 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { threadPool, List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)) ); - assertTrue(remoteClusterStateService.getRemoteRoutingTableService().isPresent()); + assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof DefaultRemoteRoutingTableService); } public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { @@ -1427,7 +1428,7 @@ public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { "test-index", "index-uuid", "routing-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() .indices(List.of(uploadedIndexMetadata)) @@ -1477,7 +1478,7 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx "test-index", "index-uuid", "routing-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() @@ -1532,7 +1533,7 @@ public void testWriteIncrementalMetadataSuccessWithRoutingTable() throws IOExcep "test-index", "index-uuid", "routing-filename", - RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() .indices(List.of(uploadedIndexMetadata)) From b1f73f1ceb9ed2f82b2409de92b9e0e8ff8b005b Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Sun, 9 Jun 2024 19:58:08 +0530 Subject: [PATCH 45/50] spotless fix Signed-off-by: Himshikha Gupta --- ...=> InternalRemoteRoutingTableService.java} | 8 +++--- .../remote/NoopRemoteRoutingTableService.java | 2 +- .../RemoteRoutingTableServiceFactory.java | 2 +- .../common/settings/ClusterSettings.java | 6 ++--- .../remote/RemoteClusterStateService.java | 6 ++--- .../index/remote/RemoteStoreEnums.java | 2 +- .../index/remote/RemoteStorePathStrategy.java | 1 + ...RemoteRoutingTableServiceFactoryTests.java | 2 +- .../RemoteRoutingTableServiceTests.java | 26 +++++++++---------- .../remote/ClusterMetadataManifestTests.java | 4 +-- .../RemoteClusterStateServiceTests.java | 10 +++---- 11 files changed, 35 insertions(+), 34 deletions(-) rename server/src/main/java/org/opensearch/cluster/routing/remote/{DefaultRemoteRoutingTableService.java => InternalRemoteRoutingTableService.java} (96%) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/DefaultRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java similarity index 96% rename from server/src/main/java/org/opensearch/cluster/routing/remote/DefaultRemoteRoutingTableService.java rename to server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java index f929c670f768c..03a60d15bde01 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/DefaultRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java @@ -59,7 +59,7 @@ * * @opensearch.internal */ -public class DefaultRemoteRoutingTableService extends AbstractLifecycleComponent implements RemoteRoutingTableService { +public class InternalRemoteRoutingTableService extends AbstractLifecycleComponent implements RemoteRoutingTableService { /** * This setting is used to set the remote routing table store blob store path type strategy. @@ -90,14 +90,14 @@ public class DefaultRemoteRoutingTableService extends AbstractLifecycleComponent public static final String DELIMITER = "__"; public static final String INDEX_ROUTING_METADATA_PREFIX = "indexRouting--"; - private static final Logger logger = LogManager.getLogger(DefaultRemoteRoutingTableService.class); + private static final Logger logger = LogManager.getLogger(InternalRemoteRoutingTableService.class); private final Settings settings; private final Supplier repositoriesService; private BlobStoreRepository blobStoreRepository; private RemoteStoreEnums.PathType pathType; private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo; - public DefaultRemoteRoutingTableService( + public InternalRemoteRoutingTableService( Supplier repositoriesService, Settings settings, ClusterSettings clusterSettings @@ -158,7 +158,7 @@ public CheckedRunnable getIndexRoutingAsyncAction( BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN); BlobPath path = pathType.path( - RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(), + RemoteStorePathStrategy.PathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(), pathHashAlgo ); final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java index 99933c72195e9..8a92e079a8dba 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java @@ -44,7 +44,7 @@ public CheckedRunnable getIndexRoutingAsyncAction( LatchedActionListener latchedActionListener, BlobPath clusterBasePath ) { - //noop + // noop return () -> {}; } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java index 41c441c635e5c..49f90fa261f27 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java @@ -34,7 +34,7 @@ public static RemoteRoutingTableService getService( ClusterSettings clusterSettings ) { if (isRemoteRoutingTableEnabled(settings)) { - return new DefaultRemoteRoutingTableService(repositoriesService, settings, clusterSettings); + return new InternalRemoteRoutingTableService(repositoriesService, settings, clusterSettings); } return new NoopRemoteRoutingTableService(); } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index c0592728c89ff..57a48dce7a575 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -77,7 +77,7 @@ import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; -import org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterManagerService; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; @@ -722,8 +722,8 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING, - DefaultRemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, - DefaultRemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, + InternalRemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, + InternalRemoteRoutingTableService.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, // Admission Control Settings AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 0b50ba7eb08b6..5142bf287cfa5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.routing.remote.RemoteRoutingTableServiceFactory; import org.opensearch.cluster.service.ClusterService; @@ -556,7 +556,7 @@ private UploadedMetadataResults writeMetadataInParallel( indicesRoutingToUpload.forEach(indexRoutingTable -> { uploadTasks.put( - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName(), + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName(), remoteRoutingTableService.getIndexRoutingAsyncAction( clusterState, indexRoutingTable, @@ -613,7 +613,7 @@ private UploadedMetadataResults writeMetadataInParallel( UploadedMetadataResults response = new UploadedMetadataResults(); results.forEach((name, uploadedMetadata) -> { if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class) - && uploadedMetadata.getComponent().contains(DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) { + && uploadedMetadata.getComponent().contains(InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) { response.uploadedIndicesRoutingMetadata.add((UploadedIndexMetadata) uploadedMetadata); } else if (name.contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom-- diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java index 17fb241846a72..b0376c97e6994 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java @@ -13,13 +13,13 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.hash.FNV1a; +import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; -import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import static java.util.Collections.unmodifiableMap; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java index 6dabf08a6fcf5..0439a8ee6ed78 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java @@ -194,6 +194,7 @@ public void assertIsValid() { + dataType + " are not supported together"; } + /** * Returns a new builder for {@link ShardDataPathInput}. */ diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java index 717610a57f8f0..d0c2cca4b46f0 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java @@ -46,6 +46,6 @@ public void testGetServiceWhenRemoteRoutingEnabled() { settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) ); - assertTrue(service instanceof DefaultRemoteRoutingTableService); + assertTrue(service instanceof InternalRemoteRoutingTableService); } } diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 7c5c9dfc27a32..8fd410e774332 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -54,8 +54,8 @@ import org.mockito.ArgumentCaptor; -import static org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService.INDEX_ROUTING_FILE_PREFIX; -import static org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN; +import static org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService.INDEX_ROUTING_FILE_PREFIX; +import static org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -73,7 +73,7 @@ public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { - private DefaultRemoteRoutingTableService remoteRoutingTableService; + private InternalRemoteRoutingTableService remoteRoutingTableService; private Supplier repositoriesServiceSupplier; private RepositoriesService repositoriesService; private BlobStoreRepository blobStoreRepository; @@ -104,7 +104,7 @@ public void setup() { basePath = BlobPath.cleanPath().add("base-path"); - remoteRoutingTableService = new DefaultRemoteRoutingTableService( + remoteRoutingTableService = new InternalRemoteRoutingTableService( repositoriesServiceSupplier, settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) @@ -121,7 +121,7 @@ public void testFailInitializationWhenRemoteRoutingDisabled() { final Settings settings = Settings.builder().build(); assertThrows( AssertionError.class, - () -> new DefaultRemoteRoutingTableService( + () -> new InternalRemoteRoutingTableService( repositoriesServiceSupplier, settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) @@ -428,7 +428,7 @@ public void testGetAllUploadedIndicesRouting() { "test-index", "index-uuid", "index-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); List allIndiceRoutingMetadata = remoteRoutingTableService @@ -443,7 +443,7 @@ public void testGetAllUploadedIndicesRoutingExistingIndexInManifest() { "test-index", "index-uuid", "index-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indicesRouting(List.of(uploadedIndexMetadata)) @@ -461,7 +461,7 @@ public void testGetAllUploadedIndicesRoutingNewIndexFromManifest() { "test-index", "index-uuid", "index-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indicesRouting(List.of(uploadedIndexMetadata)) @@ -470,7 +470,7 @@ public void testGetAllUploadedIndicesRoutingNewIndexFromManifest() { "test-index2", "index-uuid", "index-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); List allIndiceRoutingMetadata = remoteRoutingTableService @@ -486,13 +486,13 @@ public void testGetAllUploadedIndicesRoutingIndexDeleted() { "test-index", "index-uuid", "index-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata( "test-index2", "index-uuid", "index-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indicesRouting(List.of(uploadedIndexMetadata, uploadedIndexMetadata2)) @@ -510,13 +510,13 @@ public void testGetAllUploadedIndicesRoutingNoChange() { "test-index", "index-uuid", "index-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata2 = new ClusterMetadataManifest.UploadedIndexMetadata( "test-index2", "index-uuid", "index-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indicesRouting(List.of(uploadedIndexMetadata, uploadedIndexMetadata2)) diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 9705c284b862b..7cdadfe9e96c3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -12,7 +12,7 @@ import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; -import org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -371,7 +371,7 @@ public void testClusterMetadataManifestXContentV3() throws IOException { "test-index", "test-uuid", "routing-path", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index b96ba68142879..de8b2a716197e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RoutingTable; -import org.opensearch.cluster.routing.remote.DefaultRemoteRoutingTableService; +import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService; import org.opensearch.cluster.routing.remote.NoopRemoteRoutingTableService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; @@ -1412,7 +1412,7 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { threadPool, List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)) ); - assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof DefaultRemoteRoutingTableService); + assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof InternalRemoteRoutingTableService); } public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { @@ -1428,7 +1428,7 @@ public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { "test-index", "index-uuid", "routing-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() .indices(List.of(uploadedIndexMetadata)) @@ -1478,7 +1478,7 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx "test-index", "index-uuid", "routing-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() @@ -1533,7 +1533,7 @@ public void testWriteIncrementalMetadataSuccessWithRoutingTable() throws IOExcep "test-index", "index-uuid", "routing-filename", - DefaultRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() .indices(List.of(uploadedIndexMetadata)) From 1abe899365d956c8d6fa06ff6e84a8f4d202b0f9 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 10 Jun 2024 10:57:32 +0530 Subject: [PATCH 46/50] PR comments Signed-off-by: Himshikha Gupta --- .../routing/remote/NoopRemoteRoutingTableService.java | 3 +++ .../index/remote/RemoteStorePathStrategy.java | 10 ---------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java index 8a92e079a8dba..b52c00f1f8576 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java @@ -22,6 +22,9 @@ import java.util.List; import java.util.Map; +/** + * Noop impl for RemoteRoutingTableService. + */ public class NoopRemoteRoutingTableService extends AbstractLifecycleComponent implements RemoteRoutingTableService { @Override diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java index 0439a8ee6ed78..d0250790068f7 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java @@ -185,16 +185,6 @@ BlobPath fixedSubPath() { return super.fixedSubPath().add(shardId).add(dataCategory.getName()).add(dataType.getName()); } - @Override - public void assertIsValid() { - super.assertIsValid(); - assert dataCategory.isSupportedDataType(dataType) : "category:" - + dataCategory - + " type:" - + dataType - + " are not supported together"; - } - /** * Returns a new builder for {@link ShardDataPathInput}. */ From df7339caa59ff333992611df64389bce235be820 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 10 Jun 2024 11:20:56 +0530 Subject: [PATCH 47/50] Fixing gradle failure Signed-off-by: Himshikha Gupta --- .../routing/remote/InternalRemoteRoutingTableService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java index 03a60d15bde01..dcf106914c571 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java @@ -221,7 +221,7 @@ private void uploadIndex( indexRoutingInput.writeTo(streamOutput); bytesInput = streamOutput.bytes(); } catch (IOException e) { - logger.error("Failed to serialize IndexRoutingTable for {}. ", indexRouting, e); + logger.error("Failed to serialize IndexRoutingTable for [{}]: [{}]", indexRouting, e); completionListener.onFailure(e); return; } @@ -231,7 +231,7 @@ private void uploadIndex( blobContainer.writeBlob(fileName, bytesInput.streamInput(), bytesInput.length(), true); completionListener.onResponse(null); } catch (IOException e) { - logger.error("Failed to write IndexRoutingTable to remote store for indexRouting {}. ", indexRouting, e); + logger.error("Failed to write IndexRoutingTable to remote store for indexRouting [{}]: [{}]", indexRouting, e); completionListener.onFailure(e); } return; @@ -255,12 +255,12 @@ private void uploadIndex( completionListener ); } catch (IOException e) { - logger.error("Failed to write IndexRoutingTable to remote store for indexRouting {}. ", indexRouting, e); + logger.error("Failed to write IndexRoutingTable to remote store for indexRouting [{}]: [{}]", indexRouting, e); completionListener.onFailure(e); } } catch (IOException e) { logger.error( - "Failed to create transfer object for IndexRoutingTable for remote store upload for indexRouting {}. ", + "Failed to create transfer object for IndexRoutingTable for remote store upload for indexRouting [{}]: [{}]", indexRouting, e ); From abd6f9099faf1bc80be41f2bff0140817b6f1917 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 10 Jun 2024 12:02:31 +0530 Subject: [PATCH 48/50] Fixing tests Signed-off-by: Himshikha Gupta --- .../opensearch/index/remote/RemoteStorePathStrategyTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java index cfa97fc3b2251..cf5876cb5caf1 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java @@ -67,7 +67,7 @@ public void testPathInput() { assertEquals(TRANSLOG, input.dataCategory()); } - public void testVariablePath() { + public void testFixedSubPath() { RemoteStorePathStrategy.PathInput input = RemoteStorePathStrategy.PathInput.builder() .basePath(BASE_PATH) .indexUUID(INDEX_UUID) @@ -81,7 +81,7 @@ public void testVariablePath() { .dataCategory(TRANSLOG) .dataType(DATA) .build(); - assertEquals(BlobPath.cleanPath().add(INDEX_UUID).add(SHARD_ID).add(TRANSLOG.getName()).add(DATA.getName()), input.fixedSubPath()); + assertEquals(BlobPath.cleanPath().add(INDEX_UUID).add(SHARD_ID).add(TRANSLOG.getName()).add(DATA.getName()), input2.fixedSubPath()); } } From c015d561e081c7114d96389d24dc2db117b8f821 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 10 Jun 2024 13:19:35 +0530 Subject: [PATCH 49/50] add log Signed-off-by: Himshikha Gupta --- .../opensearch/gateway/remote/RemoteClusterStateService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 5142bf287cfa5..1004ae7689d71 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -223,7 +223,7 @@ public class RemoteClusterStateService implements Closeable { private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]"; private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " - + "updated : [{}], custom metadata updated : [{}]"; + + "updated : [{}], custom metadata updated : [{}], indices routing updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2; @@ -459,7 +459,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( updateCoordinationMetadata, updateSettingsMetadata, updateTemplatesMetadata, - customsToUpload.size() + customsToUpload.size(), + indicesRoutingToUpload.size() ); if (durationMillis >= slowWriteLoggingThreshold.getMillis()) { logger.warn( From 13af3009376ec130d6936ba834586e3d30447808 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 10 Jun 2024 22:28:36 +0530 Subject: [PATCH 50/50] fixing test failure due to merge Signed-off-by: Himshikha Gupta --- .../gateway/remote/RemoteClusterStateServiceTests.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 1b9adb42d6433..e91eeb82d44b9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -1440,7 +1440,8 @@ public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( "test-index", @@ -1489,7 +1490,8 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( @@ -1545,7 +1547,7 @@ public void testWriteIncrementalMetadataSuccessWithRoutingTable() throws IOExcep previousClusterState, clusterState, previousManifest - ); + ).getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( "test-index",