From a66aaafac02aebed7659b42dc0fcd69d64306b02 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Wed, 17 Apr 2024 14:07:08 +0530 Subject: [PATCH 01/23] Initial commit for index routing table manifest Signed-off-by: Bukhtawar Khan --- .../remote/ClusterMetadataManifest.java | 98 ++++++++++++++++++- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4725f40076ce2..4e5891d154de0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -35,6 +35,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. + public static final int CODEC_V2 = 2; // In Codec V2 we introduce index routing-metadata in manifest file. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -48,6 +49,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); + private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); private static long term(Object[] fields) { return (long) fields[0]; @@ -97,6 +99,10 @@ private static String globalMetadataFileName(Object[] fields) { return (String) fields[11]; } + private static List indicesRouting(Object[] fields) { + return (List) fields[12]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> new ClusterMetadataManifest( @@ -133,11 +139,31 @@ private static String globalMetadataFileName(Object[] fields) { ) ); - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V1; + private static final ConstructingObjectParser PARSER_V2 = new ConstructingObjectParser<>( + "cluster_metadata_manifest", + fields -> new ClusterMetadataManifest( + term(fields), + version(fields), + clusterUUID(fields), + stateUUID(fields), + opensearchVersion(fields), + nodeId(fields), + committed(fields), + codecVersion(fields), + globalMetadataFileName(fields), + indices(fields), + previousClusterUUID(fields), + clusterUUIDCommitted(fields), + indicesRouting(fields) + ) + ); + + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); + declareParser(PARSER_V2, CODEC_V2); } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -160,6 +186,13 @@ private static void declareParser(ConstructingObjectParser= CODEC_V2) { + parser.declareObjectArray( + ConstructingObjectParser.constructorArg(), + (p, c) -> UploadedIndexMetadata.fromXContent(p), + INDICES_ROUTING_FIELD + ); + } } private final int codecVersion; @@ -174,6 +207,7 @@ private static void declareParser(ConstructingObjectParser indicesRouting; public List getIndices() { return indices; @@ -223,6 +257,10 @@ public String getGlobalMetadataFileName() { return globalMetadataFileName; } + public List getIndicesRouting() { + return indicesRouting; + } + public ClusterMetadataManifest( long clusterTerm, long version, @@ -237,6 +275,25 @@ public ClusterMetadataManifest( String previousClusterUUID, boolean clusterUUIDCommitted ) { + this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, null); + } + + public ClusterMetadataManifest( + long clusterTerm, + long version, + String clusterUUID, + String stateUUID, + Version opensearchVersion, + String nodeId, + boolean committed, + int codecVersion, + String globalMetadataFileName, + List indices, + String previousClusterUUID, + boolean clusterUUIDCommitted, + List indicesRouting + ) { this.clusterTerm = clusterTerm; this.stateVersion = version; this.clusterUUID = clusterUUID; @@ -249,6 +306,7 @@ public ClusterMetadataManifest( this.indices = Collections.unmodifiableList(indices); this.previousClusterUUID = previousClusterUUID; this.clusterUUIDCommitted = clusterUUIDCommitted; + this.indicesRouting = Collections.unmodifiableList(indicesRouting); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -262,12 +320,18 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_2_12_0)) { + if (in.getVersion().onOrAfter(Version.V_2_14_0)) { + this.codecVersion = in.readInt(); + this.globalMetadataFileName = in.readString(); + this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); + } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.indicesRouting = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; + this.indicesRouting = null; } } @@ -301,6 +365,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } + if (onOrAfterCodecVersion(CODEC_V2)) { + builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); + { + for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { + uploadedIndexMetadata.toXContent(builder, params); + } + } + } return builder; } @@ -319,6 +391,8 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); + } else if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + out.writeCollection(indicesRouting); } } @@ -342,7 +416,8 @@ public boolean equals(Object o) { && Objects.equals(previousClusterUUID, that.previousClusterUUID) && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) - && Objects.equals(codecVersion, that.codecVersion); + && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(indicesRouting, that.indicesRouting); } @Override @@ -359,7 +434,8 @@ public int hashCode() { nodeId, committed, previousClusterUUID, - clusterUUIDCommitted + clusterUUIDCommitted, + indicesRouting ); } @@ -399,12 +475,18 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; + private List indicesRouting; public Builder indices(List indices) { this.indices = indices; return this; } + public Builder indicesRouting(List indicesRouting) { + this.indicesRouting = indicesRouting; + return this; + } + public Builder codecVersion(int codecVersion) { this.codecVersion = codecVersion; return this; @@ -454,6 +536,10 @@ public List getIndices() { return indices; } + public List getIndicesRouting() { + return indicesRouting; + } + public Builder previousClusterUUID(String previousClusterUUID) { this.previousClusterUUID = previousClusterUUID; return this; @@ -481,6 +567,7 @@ public Builder(ClusterMetadataManifest manifest) { this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; + this.indicesRouting = new ArrayList<>(manifest.indicesRouting); } public ClusterMetadataManifest build() { @@ -496,7 +583,8 @@ public ClusterMetadataManifest build() { globalMetadataFileName, indices, previousClusterUUID, - clusterUUIDCommitted + clusterUUIDCommitted, + indicesRouting ); } From ad480ee208a8637920cba04d85c8fb9c426aa96c Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 20 Apr 2024 00:00:39 +0530 Subject: [PATCH 02/23] Changes for IndexRoutingTableHeader Signed-off-by: Bukhtawar Khan --- .../stream}/BufferedChecksumStreamInput.java | 2 +- .../stream}/BufferedChecksumStreamOutput.java | 2 +- .../org/opensearch/common/util/BigArrays.java | 2 +- .../routingtable/IndexRoutingTableHeader.java | 135 ++++++++++++++++++ .../index/translog/BaseTranslogReader.java | 1 + .../opensearch/index/translog/Translog.java | 2 + .../index/translog/TranslogHeader.java | 2 + .../index/translog/TranslogSnapshot.java | 1 + .../index/translog/TranslogWriter.java | 1 + .../coordination/CoordinationStateTests.java | 2 +- .../remote/ClusterMetadataManifestTests.java | 6 +- .../snapshots/BlobStoreFormatTests.java | 2 +- 12 files changed, 150 insertions(+), 8 deletions(-) rename server/src/main/java/org/opensearch/{index/translog => common/io/stream}/BufferedChecksumStreamInput.java (99%) rename server/src/main/java/org/opensearch/{index/translog => common/io/stream}/BufferedChecksumStreamOutput.java (98%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java similarity index 99% rename from server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java rename to server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java index f75f27b7bcb91..f3341712275f9 100644 --- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.index.translog; +package org.opensearch.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.apache.lucene.util.BitUtil; diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java similarity index 98% rename from server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java rename to server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java index 9e96664c79cc5..254f228f1c739 100644 --- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.index.translog; +package org.opensearch.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java index 92371c2c77ef9..734e5535c3cf4 100644 --- a/server/src/main/java/org/opensearch/common/util/BigArrays.java +++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java @@ -49,7 +49,7 @@ import java.util.Arrays; /** - * Utility class to work with arrays. + * Utility class to work with arrays.Ø * * @opensearch.api * */ diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java new file mode 100644 index 0000000000000..87d94dc1147d1 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -0,0 +1,135 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.store.InputStreamDataInput; +import org.apache.lucene.store.OutputStreamDataOutput; +import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; + +import java.io.EOFException; +import java.io.IOException; + +/** + * The stored header information for the individual index routing table + */ +public class IndexRoutingTableHeader { + + private int routingTableVersion; + + private String indexName; + + private Version nodeVersion; + + public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; + + public static final int INITIAL_VERSION = 1; + + public static final int CURRENT_VERSION = INITIAL_VERSION; + + + public IndexRoutingTableHeader(int routingTableVersion, String indexName, Version nodeVersion) { + this.routingTableVersion = routingTableVersion; + this.indexName = indexName; + this.nodeVersion = nodeVersion; + } + + /** + * Returns the bytes reference for the {@link IndexRoutingTableHeader} + * @return the {@link BytesReference} + * @throws IOException + */ + public BytesReference write() throws IOException { + BytesReference bytesReference; + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput)) { + CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); + // Write version + out.writeInt(routingTableVersion); + out.writeInt(nodeVersion.id); + out.writeString(indexName); + // Checksum header + out.writeInt((int) out.getChecksum()); + out.flush(); + bytesReference = bytesStreamOutput.bytes(); + } + return bytesReference; + } + + + /** + * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} + * @param inBytes + * @param source + * @return + * @throws IOException + */ + public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { + try { + try(BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { + readHeaderVersion(in); + final int version = in.readInt(); + final int nodeVersion = in.readInt(); + final String name = in.readString(); + verifyChecksum(in); + assert version >= 0 : "Version must be non-negative [" + version + "]"; + assert in.readByte() == -1 : "Header is not fully read"; + return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); + } + } catch (EOFException e) { + throw new IOException("index routing header truncated", e); + } + } + + + static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { + // This absolutely must come first, or else reading the checksum becomes part of the checksum + long expectedChecksum = in.getChecksum(); + long readChecksum = Integer.toUnsignedLong(in.readInt()); + if (readChecksum != expectedChecksum) { + throw new IOException( + "checksum verification failed - expected: 0x" + + Long.toHexString(expectedChecksum) + + ", got: 0x" + + Long.toHexString(readChecksum) + ); + } + } + + static int readHeaderVersion(final StreamInput in) throws IOException { + final int version; + try { + version = CodecUtil.checkHeader(new InputStreamDataInput(in), INDEX_ROUTING_HEADER_CODEC, INITIAL_VERSION, CURRENT_VERSION); + } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { + throw new IOException("index routing table header corrupted", e); + } + return version; + } + + public int getRoutingTableVersion() { + return routingTableVersion; + } + + public String getIndexName() { + return indexName; + } + + public Version getNodeVersion() { + return nodeVersion; + } +} diff --git a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java index d6fa2a2e53de3..9088eb6b20fb8 100644 --- a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java +++ b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java @@ -32,6 +32,7 @@ package org.opensearch.index.translog; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 842e9c77d2350..18b10cb886996 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -38,6 +38,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index 7b5be9505f27a..c622214a3a69d 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -40,6 +40,8 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.Channels; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java index 89718156cbbe8..a6e322bf58fff 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java @@ -32,6 +32,7 @@ package org.opensearch.index.translog; import org.opensearch.common.io.Channels; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.index.seqno.SequenceNumbers; import java.io.EOFException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index 86f7567f3333d..da3c7a8dee219 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -42,6 +42,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Channels; import org.opensearch.common.io.DiskIoBufferPool; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 1c0dc7fc1ca2d..0fe52d2e72d4b 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -942,7 +942,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep randomAlphaOfLength(10), Collections.emptyList(), randomAlphaOfLength(10), - true + true, ); Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 6c9a3201656d7..647f4c913931f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -40,7 +40,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true + true, ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -67,7 +67,7 @@ public void testClusterMetadataManifestXContent() throws IOException { "test-global-metadata-file", Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true + true, ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -93,7 +93,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "test-global-metadata-file", randomUploadedIndexMetadataList(), "yfObdx8KSMKKrXf8UyHhM", - true + true, ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index c5f36fcc01983..8c64f9a3170b2 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -55,7 +55,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.translog.BufferedChecksumStreamOutput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.test.OpenSearchTestCase; From 2ad70c4fb9df43d99ce3714df1b4303f7589dc9e Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 20 Apr 2024 00:04:24 +0530 Subject: [PATCH 03/23] Revert unintentional changes for IndexRoutingTableHeader Signed-off-by: Bukhtawar Khan --- .../src/main/java/org/opensearch/common/util/BigArrays.java | 2 +- .../cluster/coordination/CoordinationStateTests.java | 2 +- .../gateway/remote/ClusterMetadataManifestTests.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java index 734e5535c3cf4..92371c2c77ef9 100644 --- a/server/src/main/java/org/opensearch/common/util/BigArrays.java +++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java @@ -49,7 +49,7 @@ import java.util.Arrays; /** - * Utility class to work with arrays.Ø + * Utility class to work with arrays. * * @opensearch.api * */ diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 0fe52d2e72d4b..1c0dc7fc1ca2d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -942,7 +942,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep randomAlphaOfLength(10), Collections.emptyList(), randomAlphaOfLength(10), - true, + true ); Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 647f4c913931f..6c9a3201656d7 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -40,7 +40,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true, + true ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -67,7 +67,7 @@ public void testClusterMetadataManifestXContent() throws IOException { "test-global-metadata-file", Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true, + true ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -93,7 +93,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "test-global-metadata-file", randomUploadedIndexMetadataList(), "yfObdx8KSMKKrXf8UyHhM", - true, + true ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( From acc172ea899fe2e0c5a75fe1a030b081af14aaca Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 20 Apr 2024 00:04:24 +0530 Subject: [PATCH 04/23] Revert unintentional changes for IndexRoutingTableHeader Signed-off-by: Bukhtawar Khan From f65b102996fd23e405ce4ea3bdfcdc54111cfe21 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sun, 21 Apr 2024 00:15:47 +0530 Subject: [PATCH 05/23] Changes for IndexRoutingTableInputStream Signed-off-by: Bukhtawar Khan --- .../routingtable/IndexRoutingTableHeader.java | 27 ++- .../IndexRoutingTableInputStream.java | 157 ++++++++++++++++++ 2 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 87d94dc1147d1..04a0f1868b64f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -15,12 +15,12 @@ import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; import org.opensearch.Version; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import java.io.EOFException; import java.io.IOException; @@ -30,11 +30,11 @@ */ public class IndexRoutingTableHeader { - private int routingTableVersion; + private final long routingTableVersion; - private String indexName; + private final String indexName; - private Version nodeVersion; + private final Version nodeVersion; public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; @@ -42,8 +42,7 @@ public class IndexRoutingTableHeader { public static final int CURRENT_VERSION = INITIAL_VERSION; - - public IndexRoutingTableHeader(int routingTableVersion, String indexName, Version nodeVersion) { + public IndexRoutingTableHeader(long routingTableVersion, String indexName, Version nodeVersion) { this.routingTableVersion = routingTableVersion; this.indexName = indexName; this.nodeVersion = nodeVersion; @@ -56,11 +55,13 @@ public IndexRoutingTableHeader(int routingTableVersion, String indexName, Versio */ public BytesReference write() throws IOException { BytesReference bytesReference; - try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput)) { + try ( + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) + ) { CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); // Write version - out.writeInt(routingTableVersion); + out.writeLong(routingTableVersion); out.writeInt(nodeVersion.id); out.writeString(indexName); // Checksum header @@ -71,7 +72,6 @@ public BytesReference write() throws IOException { return bytesReference; } - /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} * @param inBytes @@ -81,7 +81,7 @@ public BytesReference write() throws IOException { */ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { try { - try(BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { + try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); final int version = in.readInt(); final int nodeVersion = in.readInt(); @@ -96,7 +96,6 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce } } - static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { // This absolutely must come first, or else reading the checksum becomes part of the checksum long expectedChecksum = in.getChecksum(); @@ -121,7 +120,7 @@ static int readHeaderVersion(final StreamInput in) throws IOException { return version; } - public int getRoutingTableVersion() { + public long getRoutingTableVersion() { return routingTableVersion; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java new file mode 100644 index 0000000000000..ac65232e9a24d --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.opensearch.Version; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; + +public class IndexRoutingTableInputStream extends InputStream { + + /** + * The buffer where data is stored. + */ + protected byte[] buf; + + /** + * The number of valid bytes in the buffer. + */ + protected int count; + + /** + * The buffer left over from the last fill + */ + protected byte[] leftOverBuf; + + /** + * The mark position + */ + protected int markPos = -1; + + /** + * The read limit + */ + protected int markLimit; + + /** + * The position + */ + protected int pos; + + private static final int BUFFER_SIZE = 8192; + + private final IndexRoutingTableHeader indexRoutingTableHeader; + + private final Iterator shardIter; + + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { + this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); + } + + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion, int size) + throws IOException { + this.buf = new byte[size]; + this.shardIter = indexRoutingTable.iterator(); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); + initialFill(); + } + + @Override + public int read() throws IOException { + if (pos >= count) { + maybeResizeAndFill(); + if (pos >= count) return -1; + } + return buf[pos++] & 0xff; + } + + private void initialFill() throws IOException { + BytesReference bytesReference = indexRoutingTableHeader.write(); + buf = bytesReference.toBytesRef().bytes; + count = bytesReference.length(); + fill(buf); + } + + private void fill(byte[] buf) throws IOException { + if (leftOverBuf != null) { + System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + } + if (count < buf.length && shardIter.hasNext()) { + IndexShardRoutingTable next = shardIter.next(); + BytesReference bytesRef; + try ( + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) + ) { + IndexShardRoutingTable.Builder.writeTo(next, out); + // Checksum header + out.writeInt((int) out.getChecksum()); + out.flush(); + bytesRef = bytesStreamOutput.bytes(); + } + if (bytesRef.length() < buf.length - count) { + System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, bytesRef.length()); + count += bytesRef.length(); + leftOverBuf = null; + } else { + System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); + count += buf.length - count; + leftOverBuf = new byte[bytesRef.length() - count]; + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count + 1, leftOverBuf, 0, bytesRef.length() - count); + } + } + } + + private void maybeResizeAndFill() throws IOException { + byte[] buffer = buf; + if (markPos == -1) pos = 0; /* no mark: throw away the buffer */ + else if (pos >= buffer.length) { /* no room left in buffer */ + if (markPos > 0) { /* can throw away early part of the buffer */ + int sz = pos - markPos; + System.arraycopy(buffer, markPos, buffer, 0, sz); + pos = sz; + markPos = 0; + } else if (buffer.length >= markLimit) { + markPos = -1; /* buffer got too big, invalidate mark */ + pos = 0; /* drop buffer contents */ + } else { /* grow buffer */ + int nsz = markLimit + 1; + byte[] nbuf = new byte[nsz]; + System.arraycopy(buffer, 0, nbuf, 0, pos); + buffer = nbuf; + } + } + count = pos; + fill(buffer); + } + + @Override + public void mark(int readlimit) { + markLimit = readlimit; + markPos = pos; + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void reset() throws IOException { + if (markPos < 0) throw new IOException("Resetting to invalid mark"); + pos = markPos; + } +} From 441f52023d3757e1073fefb629fe145f19ff671f Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 7 May 2024 16:11:06 +0530 Subject: [PATCH 06/23] Fixing IndexRoutingTableInputStream and moving checksum to end to file Signed-off-by: Himshikha Gupta --- .../remote/ClusterMetadataManifest.java | 15 +++-- .../routingtable/IndexRoutingTableHeader.java | 32 ++--------- .../IndexRoutingTableInputStream.java | 57 ++++++++++++------- .../IndexRoutingTableHeaderTests.java | 35 ++++++++++++ 4 files changed, 86 insertions(+), 53 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4e5891d154de0..0279f8e0fd805 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -276,7 +276,7 @@ public ClusterMetadataManifest( boolean clusterUUIDCommitted ) { this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, - globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, null); + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, new ArrayList<>()); } public ClusterMetadataManifest( @@ -355,7 +355,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startArray(INDICES_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indices) { + builder.startObject(); uploadedIndexMetadata.toXContent(builder, params); + builder.endObject(); } } builder.endArray(); @@ -369,9 +371,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { + builder.startObject(); uploadedIndexMetadata.toXContent(builder, params); + builder.endObject(); } } + builder.endArray(); } return builder; } @@ -391,7 +396,8 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); - } else if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + } + if (out.getVersion().onOrAfter(Version.V_2_14_0)) { out.writeCollection(indicesRouting); } } @@ -659,11 +665,10 @@ public String getIndexUUID() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject() + return builder .field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) .field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID()) - .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()) - .endObject(); + .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 04a0f1868b64f..23ab700d5a34f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -21,6 +21,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.EOFException; import java.io.IOException; @@ -50,26 +51,16 @@ public IndexRoutingTableHeader(long routingTableVersion, String indexName, Versi /** * Returns the bytes reference for the {@link IndexRoutingTableHeader} - * @return the {@link BytesReference} * @throws IOException */ - public BytesReference write() throws IOException { - BytesReference bytesReference; - try ( - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) - ) { + public void write(StreamOutput out) throws IOException { CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); // Write version out.writeLong(routingTableVersion); out.writeInt(nodeVersion.id); out.writeString(indexName); - // Checksum header - out.writeInt((int) out.getChecksum()); + out.flush(); - bytesReference = bytesStreamOutput.bytes(); - } - return bytesReference; } /** @@ -83,10 +74,9 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); - final int version = in.readInt(); + final long version = in.readLong(); final int nodeVersion = in.readInt(); final String name = in.readString(); - verifyChecksum(in); assert version >= 0 : "Version must be non-negative [" + version + "]"; assert in.readByte() == -1 : "Header is not fully read"; return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); @@ -96,20 +86,6 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce } } - static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { - // This absolutely must come first, or else reading the checksum becomes part of the checksum - long expectedChecksum = in.getChecksum(); - long readChecksum = Integer.toUnsignedLong(in.readInt()); - if (readChecksum != expectedChecksum) { - throw new IOException( - "checksum verification failed - expected: 0x" - + Long.toHexString(expectedChecksum) - + ", got: 0x" - + Long.toHexString(readChecksum) - ); - } - } - static int readHeaderVersion(final StreamInput in) throws IOException { final int version; try { diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index ac65232e9a24d..40e5908d5c65c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -54,8 +54,9 @@ public class IndexRoutingTableInputStream extends InputStream { private static final int BUFFER_SIZE = 8192; private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; + private final BytesStreamOutput bytesStreamOutput; + private final BufferedChecksumStreamOutput out; public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); @@ -66,7 +67,10 @@ public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long ve this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); - initialFill(); + this.bytesStreamOutput = new BytesStreamOutput(); + this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); + + initialFill(indexRoutingTable.shards().size()); } @Override @@ -78,39 +82,52 @@ public int read() throws IOException { return buf[pos++] & 0xff; } - private void initialFill() throws IOException { - BytesReference bytesReference = indexRoutingTableHeader.write(); - buf = bytesReference.toBytesRef().bytes; - count = bytesReference.length(); + private void initialFill(int shardCount) throws IOException { + indexRoutingTableHeader.write(out); + out.writeVInt(shardCount); + + System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0 , buf, 0, bytesStreamOutput.bytes().length()); + count = bytesStreamOutput.bytes().length(); + bytesStreamOutput.reset(); fill(buf); } private void fill(byte[] buf) throws IOException { if (leftOverBuf != null) { - System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + if(leftOverBuf.length > buf.length - count) { + // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in leftOverBuf. + System.arraycopy(leftOverBuf, 0, buf, count, buf.length - count); + byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; + System.arraycopy(leftOverBuf, buf.length - count , tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); + leftOverBuf = tempLeftOverBuffer; + count = buf.length - count; + } else { + System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + count += leftOverBuf.length; + leftOverBuf = null; + } } + if (count < buf.length && shardIter.hasNext()) { IndexShardRoutingTable next = shardIter.next(); - BytesReference bytesRef; - try ( - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) - ) { - IndexShardRoutingTable.Builder.writeTo(next, out); - // Checksum header - out.writeInt((int) out.getChecksum()); - out.flush(); - bytesRef = bytesStreamOutput.bytes(); + IndexShardRoutingTable.Builder.writeTo(next, out); + //Add checksum for the file after all shards are done + if(!shardIter.hasNext()) { + out.writeLong(out.getChecksum()); } + out.flush(); + BytesReference bytesRef = bytesStreamOutput.bytes(); + bytesStreamOutput.reset(); + if (bytesRef.length() < buf.length - count) { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, bytesRef.length()); count += bytesRef.length(); leftOverBuf = null; } else { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); - count += buf.length - count; - leftOverBuf = new byte[bytesRef.length() - count]; - System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count + 1, leftOverBuf, 0, bytesRef.length() - count); + leftOverBuf = new byte[bytesRef.length() - (buf.length - count)]; + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count , leftOverBuf, 0, bytesRef.length() - (buf.length - count)); + count = buf.length; } } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java new file mode 100644 index 0000000000000..068db554b4226 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.InputStreamDataInput; +import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { + + public void testWrite() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + IndexRoutingTableHeader header = new IndexRoutingTableHeader(1, "dummyIndex", Version.V_3_0_0); + header.write(out); + + BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); + CodecUtil.checkHeader(new InputStreamDataInput(in),IndexRoutingTableHeader.INDEX_ROUTING_HEADER_CODEC, IndexRoutingTableHeader.INITIAL_VERSION, IndexRoutingTableHeader.CURRENT_VERSION ); + assertEquals(1, in.readLong()); + assertEquals(Version.V_3_0_0.id, in.readInt()); + assertEquals("dummyIndex", in.readString()); + } + +} From ee70dca9af636bdafae507fe5b004a7de4708f20 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Wed, 8 May 2024 11:09:54 +0530 Subject: [PATCH 07/23] Add read flow for IndexRoutingTable Signed-off-by: Arpit Bandejiya --- .../routingtable/IndexRoutingTableHeader.java | 10 +-- .../IndexRoutingTableInputStreamReader.java | 77 +++++++++++++++++++ .../IndexRoutingTableInputStreamTests.java | 61 +++++++++++++++ 3 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 23ab700d5a34f..e29ce5a79dc02 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -65,22 +65,18 @@ public void write(StreamOutput out) throws IOException { /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} - * @param inBytes - * @param source - * @return + * @param in + * @return IndexRoutingTableHeader * @throws IOException */ - public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { + public static IndexRoutingTableHeader read(BufferedChecksumStreamInput in) throws IOException { try { - try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); final long version = in.readLong(); final int nodeVersion = in.readInt(); final String name = in.readString(); assert version >= 0 : "Version must be non-negative [" + version + "]"; - assert in.readByte() == -1 : "Header is not fully read"; return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); - } } catch (EOFException e) { throw new IOException("index routing header truncated", e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java new file mode 100644 index 0000000000000..35ae9f287d7f2 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IndexRoutingTableInputStreamReader { + + private final StreamInput streamInput; + + private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStreamReader.class); + + public IndexRoutingTableInputStreamReader(InputStream inputStream) throws IOException { + this.streamInput = new InputStreamStreamInput(inputStream); + } + + public Map read() throws IOException { + try { + try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(streamInput, "assertion")) { + // Read the Table Header first + IndexRoutingTableHeader.read(in); + int shards = in.readVInt(); + logger.info("Number of Index Routing Table {}", shards); + Map indicesRouting = new HashMap(Collections.EMPTY_MAP); + for(int i=0; i { + try { + logger.info("IndexShardRoutingTables: {}", indexShardRoutingTables); + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables, + initialRoutingTable.version(), Version.CURRENT); + + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + Map indexShardRoutingTableMap = reader.read(); + + logger.info("indexShardRoutingTableMap: {}", indexShardRoutingTableMap); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + +} From 7aeecc84592d3948b932c5973cf218091297237f Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 13 May 2024 18:08:24 +0530 Subject: [PATCH 08/23] Moving routing table version from IndexRouting stream to manifest Signed-off-by: Himshikha Gupta --- .../remote/ClusterMetadataManifest.java | 33 ++++++++++- .../routingtable/IndexRoutingTableHeader.java | 59 +++++-------------- .../IndexRoutingTableInputStream.java | 9 ++- .../remote/ClusterMetadataManifestTests.java | 29 +++++++++ .../IndexRoutingTableHeaderTests.java | 14 ++--- .../IndexRoutingTableInputStreamTests.java | 12 ++-- 6 files changed, 90 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 0279f8e0fd805..5b7192c624fed 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -49,6 +49,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); + private static final ParseField ROUTING_TABLE_VERSION_FIELD = new ParseField("routing_table_version"); private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); private static long term(Object[] fields) { @@ -99,8 +100,12 @@ private static String globalMetadataFileName(Object[] fields) { return (String) fields[11]; } + private static long routingTableVersion(Object[] fields) { + return (long) fields[12]; + } + private static List indicesRouting(Object[] fields) { - return (List) fields[12]; + return (List) fields[13]; } private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( @@ -154,6 +159,7 @@ private static List indicesRouting(Object[] fields) { indices(fields), previousClusterUUID(fields), clusterUUIDCommitted(fields), + routingTableVersion(fields), indicesRouting(fields) ) ); @@ -187,6 +193,7 @@ private static void declareParser(ConstructingObjectParser= CODEC_V2) { + parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD); parser.declareObjectArray( ConstructingObjectParser.constructorArg(), (p, c) -> UploadedIndexMetadata.fromXContent(p), @@ -207,6 +214,7 @@ private static void declareParser(ConstructingObjectParser indicesRouting; public List getIndices() { @@ -257,6 +265,10 @@ public String getGlobalMetadataFileName() { return globalMetadataFileName; } + public long getRoutingTableVersion() { + return routingTableVersion; + } + public List getIndicesRouting() { return indicesRouting; } @@ -276,7 +288,7 @@ public ClusterMetadataManifest( boolean clusterUUIDCommitted ) { this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, - globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, new ArrayList<>()); + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, -1, new ArrayList<>()); } public ClusterMetadataManifest( @@ -292,6 +304,7 @@ public ClusterMetadataManifest( List indices, String previousClusterUUID, boolean clusterUUIDCommitted, + long routingTableVersion, List indicesRouting ) { this.clusterTerm = clusterTerm; @@ -306,6 +319,7 @@ public ClusterMetadataManifest( this.indices = Collections.unmodifiableList(indices); this.previousClusterUUID = previousClusterUUID; this.clusterUUIDCommitted = clusterUUIDCommitted; + this.routingTableVersion = routingTableVersion; this.indicesRouting = Collections.unmodifiableList(indicesRouting); } @@ -323,14 +337,17 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_14_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.routingTableVersion = in.readLong(); this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.routingTableVersion = -1; this.indicesRouting = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; + this.routingTableVersion = -1; this.indicesRouting = null; } } @@ -368,6 +385,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } if (onOrAfterCodecVersion(CODEC_V2)) { + builder.field(ROUTING_TABLE_VERSION_FIELD.getPreferredName(), getRoutingTableVersion()); builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { @@ -398,6 +416,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(globalMetadataFileName); } if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + out.writeLong(routingTableVersion); out.writeCollection(indicesRouting); } } @@ -423,6 +442,7 @@ public boolean equals(Object o) { && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(routingTableVersion, that.routingTableVersion) && Objects.equals(indicesRouting, that.indicesRouting); } @@ -441,6 +461,7 @@ public int hashCode() { committed, previousClusterUUID, clusterUUIDCommitted, + routingTableVersion, indicesRouting ); } @@ -481,6 +502,7 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; + private long routingTableVersion; private List indicesRouting; public Builder indices(List indices) { @@ -488,6 +510,11 @@ public Builder indices(List indices) { return this; } + public Builder routingTableVersion(long routingTableVersion) { + this.routingTableVersion = routingTableVersion; + return this; + } + public Builder indicesRouting(List indicesRouting) { this.indicesRouting = indicesRouting; return this; @@ -573,6 +600,7 @@ public Builder(ClusterMetadataManifest manifest) { this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; + this.routingTableVersion = manifest.routingTableVersion; this.indicesRouting = new ArrayList<>(manifest.indicesRouting); } @@ -590,6 +618,7 @@ public ClusterMetadataManifest build() { indices, previousClusterUUID, clusterUUIDCommitted, + routingTableVersion, indicesRouting ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index e29ce5a79dc02..bc99fea9b5c09 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -14,12 +14,6 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; -import org.opensearch.Version; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -31,52 +25,27 @@ */ public class IndexRoutingTableHeader { - private final long routingTableVersion; - - private final String indexName; - - private final Version nodeVersion; - public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; - public static final int INITIAL_VERSION = 1; - public static final int CURRENT_VERSION = INITIAL_VERSION; + private final String indexName; - public IndexRoutingTableHeader(long routingTableVersion, String indexName, Version nodeVersion) { - this.routingTableVersion = routingTableVersion; + public IndexRoutingTableHeader(String indexName) { this.indexName = indexName; - this.nodeVersion = nodeVersion; - } - - /** - * Returns the bytes reference for the {@link IndexRoutingTableHeader} - * @throws IOException - */ - public void write(StreamOutput out) throws IOException { - CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); - // Write version - out.writeLong(routingTableVersion); - out.writeInt(nodeVersion.id); - out.writeString(indexName); - - out.flush(); } /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} + * * @param in * @return IndexRoutingTableHeader * @throws IOException */ - public static IndexRoutingTableHeader read(BufferedChecksumStreamInput in) throws IOException { + public static IndexRoutingTableHeader read(StreamInput in) throws IOException { try { - readHeaderVersion(in); - final long version = in.readLong(); - final int nodeVersion = in.readInt(); - final String name = in.readString(); - assert version >= 0 : "Version must be non-negative [" + version + "]"; - return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); + readHeaderVersion(in); + final String name = in.readString(); + return new IndexRoutingTableHeader(name); } catch (EOFException e) { throw new IOException("index routing header truncated", e); } @@ -92,15 +61,19 @@ static int readHeaderVersion(final StreamInput in) throws IOException { return version; } - public long getRoutingTableVersion() { - return routingTableVersion; + /** + * Returns the bytes reference for the {@link IndexRoutingTableHeader} + * + * @throws IOException + */ + public void write(StreamOutput out) throws IOException { + CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); + out.writeString(indexName); + out.flush(); } public String getIndexName() { return indexName; } - public Version getNodeVersion() { - return nodeVersion; - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index 40e5908d5c65c..5332a8a87c9a4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -8,7 +8,6 @@ package org.opensearch.gateway.remote.routingtable; -import org.opensearch.Version; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; @@ -58,15 +57,15 @@ public class IndexRoutingTableInputStream extends InputStream { private final BytesStreamOutput bytesStreamOutput; private final BufferedChecksumStreamOutput out; - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { - this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable) throws IOException { + this(indexRoutingTable, BUFFER_SIZE); } - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion, int size) + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) throws IOException { this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); this.bytesStreamOutput = new BytesStreamOutput(); this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 6c9a3201656d7..73933141ddf23 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -233,6 +233,35 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { } } + public void testClusterMetadataManifestXContentV2() throws IOException { + UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( + 1L, + 1L, + "test-cluster-uuid", + "test-state-uuid", + Version.CURRENT, + "test-node-id", + false, + ClusterMetadataManifest.CODEC_V2, + "test-metadata", + Collections.singletonList(uploadedIndexMetadata), + "prev-cluster-uuid", + true, + 1L, + Collections.singletonList(uploadedIndexMetadata) + ); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); + assertEquals(originalManifest, fromXContentManifest); + } + } + private List randomUploadedIndexMetadataList() { final int size = randomIntBetween(1, 10); final List uploadedIndexMetadataList = new ArrayList<>(size); diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index 068db554b4226..0f42508615a26 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -8,28 +8,22 @@ package org.opensearch.gateway.remote.routingtable; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.InputStreamDataInput; -import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.BytesStreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { - public void testWrite() throws IOException { + public void testIndexRoutingTableHeader() throws IOException { + IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); BytesStreamOutput out = new BytesStreamOutput(); - IndexRoutingTableHeader header = new IndexRoutingTableHeader(1, "dummyIndex", Version.V_3_0_0); header.write(out); BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); - CodecUtil.checkHeader(new InputStreamDataInput(in),IndexRoutingTableHeader.INDEX_ROUTING_HEADER_CODEC, IndexRoutingTableHeader.INITIAL_VERSION, IndexRoutingTableHeader.CURRENT_VERSION ); - assertEquals(1, in.readLong()); - assertEquals(Version.V_3_0_0.id, in.readInt()); - assertEquals("dummyIndex", in.readString()); + IndexRoutingTableHeader headerRead = IndexRoutingTableHeader.read(in); + assertEquals("dummyIndex", headerRead.getIndexName()); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java index 3957d4602201d..b10a60943798a 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java @@ -33,9 +33,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; -public class IndexRoutingTableInputStreamTests extends ReplicationTrackerTestCase { +public class IndexRoutingTableInputStreamTests extends OpenSearchTestCase { - public void testRoutingTableInputStream() throws IOException { + public void testRoutingTableInputStream(){ Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) .build(); @@ -44,14 +44,14 @@ public void testRoutingTableInputStream() throws IOException { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - logger.info("IndexShardRoutingTables: {}", indexShardRoutingTables); - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables, - initialRoutingTable.version(), Version.CURRENT); + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); Map indexShardRoutingTableMap = reader.read(); - logger.info("indexShardRoutingTableMap: {}", indexShardRoutingTableMap); + assertEquals(1, indexShardRoutingTableMap.size()); + assertNotNull(indexShardRoutingTableMap.get("test")); + assertEquals(2,indexShardRoutingTableMap.get("test").shards().size()); } catch (IOException e) { throw new RuntimeException(e); } From 7b2bc79266e4442a30c2daf8730ea0c615f1ec48 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Tue, 14 May 2024 13:30:14 +0530 Subject: [PATCH 09/23] Refactor reader and add failure test Signed-off-by: Arpit Bandejiya --- .../remote/ClusterMetadataManifest.java | 25 ++++++--- .../IndexRoutingTableInputStream.java | 22 ++++---- .../IndexRoutingTableInputStreamReader.java | 35 +++++------- .../IndexRoutingTableInputStreamTests.java | 54 +++++++++++-------- .../snapshots/BlobStoreFormatTests.java | 2 +- 5 files changed, 78 insertions(+), 60 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 5b7192c624fed..90ab4869ad1fd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -161,7 +161,7 @@ private static List indicesRouting(Object[] fields) { clusterUUIDCommitted(fields), routingTableVersion(fields), indicesRouting(fields) - ) + ) ); private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; @@ -287,8 +287,22 @@ public ClusterMetadataManifest( String previousClusterUUID, boolean clusterUUIDCommitted ) { - this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, - globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, -1, new ArrayList<>()); + this( + clusterTerm, + version, + clusterUUID, + stateUUID, + opensearchVersion, + nodeId, + committed, + codecVersion, + globalMetadataFileName, + indices, + previousClusterUUID, + clusterUUIDCommitted, + -1, + new ArrayList<>() + ); } public ClusterMetadataManifest( @@ -306,7 +320,7 @@ public ClusterMetadataManifest( boolean clusterUUIDCommitted, long routingTableVersion, List indicesRouting - ) { + ) { this.clusterTerm = clusterTerm; this.stateVersion = version; this.clusterUUID = clusterUUID; @@ -694,8 +708,7 @@ public String getIndexUUID() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder - .field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) + return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) .field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID()) .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index 5332a8a87c9a4..d4e2594bf153c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -61,8 +61,7 @@ public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable) throws this(indexRoutingTable, BUFFER_SIZE); } - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) - throws IOException { + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) throws IOException { this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); @@ -85,7 +84,7 @@ private void initialFill(int shardCount) throws IOException { indexRoutingTableHeader.write(out); out.writeVInt(shardCount); - System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0 , buf, 0, bytesStreamOutput.bytes().length()); + System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0, buf, 0, bytesStreamOutput.bytes().length()); count = bytesStreamOutput.bytes().length(); bytesStreamOutput.reset(); fill(buf); @@ -93,16 +92,17 @@ private void initialFill(int shardCount) throws IOException { private void fill(byte[] buf) throws IOException { if (leftOverBuf != null) { - if(leftOverBuf.length > buf.length - count) { - // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in leftOverBuf. + if (leftOverBuf.length > buf.length - count) { + // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in + // leftOverBuf. System.arraycopy(leftOverBuf, 0, buf, count, buf.length - count); - byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; - System.arraycopy(leftOverBuf, buf.length - count , tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); + byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; + System.arraycopy(leftOverBuf, buf.length - count, tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); leftOverBuf = tempLeftOverBuffer; count = buf.length - count; } else { System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); - count += leftOverBuf.length; + count += leftOverBuf.length; leftOverBuf = null; } } @@ -110,8 +110,8 @@ private void fill(byte[] buf) throws IOException { if (count < buf.length && shardIter.hasNext()) { IndexShardRoutingTable next = shardIter.next(); IndexShardRoutingTable.Builder.writeTo(next, out); - //Add checksum for the file after all shards are done - if(!shardIter.hasNext()) { + // Add checksum for the file after all shards are done + if (!shardIter.hasNext()) { out.writeLong(out.getChecksum()); } out.flush(); @@ -125,7 +125,7 @@ private void fill(byte[] buf) throws IOException { } else { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); leftOverBuf = new byte[bytesRef.length() - (buf.length - count)]; - System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count , leftOverBuf, 0, bytesRef.length() - (buf.length - count)); + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count, leftOverBuf, 0, bytesRef.length() - (buf.length - count)); count = buf.length; } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java index 35ae9f287d7f2..e2b4f5da5f6e9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -13,19 +13,13 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; -import java.io.BufferedReader; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; public class IndexRoutingTableInputStreamReader { @@ -34,32 +28,31 @@ public class IndexRoutingTableInputStreamReader { private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStreamReader.class); public IndexRoutingTableInputStreamReader(InputStream inputStream) throws IOException { - this.streamInput = new InputStreamStreamInput(inputStream); + streamInput = new InputStreamStreamInput(inputStream); } - public Map read() throws IOException { + public IndexRoutingTable readIndexRoutingTable(Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(streamInput, "assertion")) { - // Read the Table Header first - IndexRoutingTableHeader.read(in); - int shards = in.readVInt(); - logger.info("Number of Index Routing Table {}", shards); - Map indicesRouting = new HashMap(Collections.EMPTY_MAP); - for(int i=0; i indexShardRoutingTableMap = reader.read(); + IndexRoutingTable indexRoutingTable = reader.readIndexRoutingTable(metadata.index("test").getIndex()); - assertEquals(1, indexShardRoutingTableMap.size()); - assertNotNull(indexShardRoutingTableMap.get("test")); - assertEquals(2,indexShardRoutingTableMap.get("test").shards().size()); + assertEquals(1, indexRoutingTable.getShards().size()); + assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); + assertEquals(indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), 2); } catch (IOException e) { throw new RuntimeException(e); } }); } + public void testRoutingTableInputStreamWithInvalidIndex() { + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetadata.builder("invalid-index").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); + AtomicInteger assertionError = new AtomicInteger(); + initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { + try { + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); + + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + reader.readIndexRoutingTable(metadata.index("invalid-index").getIndex()); + + } catch (AssertionError e) { + assertionError.getAndIncrement(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + assertEquals(1, assertionError.get()); + } + } diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 8c64f9a3170b2..8d9a444c6f9e2 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -46,6 +46,7 @@ import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; @@ -55,7 +56,6 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.test.OpenSearchTestCase; From 88d266fb971d864943995b9a3e19270507ad4c11 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Tue, 14 May 2024 14:20:57 +0530 Subject: [PATCH 10/23] Fix GatewayMetaStatePersistedStateTests Signed-off-by: Arpit Bandejiya --- .../org/opensearch/gateway/remote/ClusterMetadataManifest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 90ab4869ad1fd..ec2e9d4169ca4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -599,6 +599,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { public Builder() { indices = new ArrayList<>(); + indicesRouting = new ArrayList<>(); } public Builder(ClusterMetadataManifest manifest) { From 4c9786925f539e543d7c8177c4c37a7f965a0a08 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 14 May 2024 16:15:09 +0530 Subject: [PATCH 11/23] Moving codec to version 2 for compatibility with manifest parser Signed-off-by: Himshikha Gupta --- .../gateway/remote/ClusterMetadataManifest.java | 4 ++++ .../gateway/remote/RemoteClusterStateService.java | 12 ++++++++++-- .../gateway/remote/ClusterMetadataManifestTests.java | 4 ++-- .../remote/RemoteClusterStateServiceTests.java | 8 +++++--- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index ec2e9d4169ca4..03512dfbf527f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -493,6 +493,10 @@ public static ClusterMetadataManifest fromXContentV0(XContentParser parser) thro return PARSER_V0.parse(parser, null); } + public static ClusterMetadataManifest fromXContentV1(XContentParser parser) throws IOException { + return PARSER_V1.parse(parser, null); + } + public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException { return CURRENT_PARSER.parse(parser, null); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index eaf607564185c..619ff7d6692ba 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -129,7 +129,13 @@ public class RemoteClusterStateService implements Closeable { new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); /** - * Manifest format compatible with codec v1, where we introduced codec versions/global metadata. + * Manifest format compatible with older codec v1, where codec versions/global metadata was introduced. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); + + /** + * Manifest format compatible with codec v2, where we introduced routing table metadata in manifest. */ public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( "cluster-metadata-manifest", @@ -172,7 +178,7 @@ public class RemoteClusterStateService implements Closeable { private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private final RemotePersistenceStats remoteStateStats; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; // ToXContent Params with gateway mode. @@ -1175,6 +1181,8 @@ private ChecksumBlobStoreFormat getClusterMetadataManif long codecVersion = getManifestCodecVersion(fileName); if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V1; } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { return CLUSTER_METADATA_MANIFEST_FORMAT_V0; } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 73933141ddf23..f9ba6415bf59f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -53,7 +53,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { } } - public void testClusterMetadataManifestXContent() throws IOException { + public void testClusterMetadataManifestXContentV1() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( 1L, @@ -75,7 +75,7 @@ public void testClusterMetadataManifestXContent() throws IOException { builder.endObject(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV1(parser); assertEquals(originalManifest, fromXContentManifest); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 9f321cd62847c..fd8d0c8bfc4a8 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -446,7 +446,7 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { * In final manifest codec version should be 1 and * global metadata should be updated, even if it was not changed in this cluster state update */ - public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOException { + public void testMigrationFromCodecV0ManifestToCodecV2Manifest() throws IOException { mockBlobStoreObjects(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -481,7 +481,7 @@ public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOExcepti // global metadata is updated assertThat(manifestAfterUpdate.getGlobalMetadataFileName(), notNullValue()); // Manifest file with codec version with 1 is updated. - assertThat(manifestAfterUpdate.getCodecVersion(), is(ClusterMetadataManifest.CODEC_V1)); + assertThat(manifestAfterUpdate.getCodecVersion(), is(ClusterMetadataManifest.CODEC_V2)); } public void testWriteIncrementalGlobalMetadataSuccess() throws IOException { @@ -803,6 +803,8 @@ public void testReadGlobalMetadata() throws IOException { .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of()) .build(); Metadata expactedMetadata = Metadata.builder().persistentSettings(Settings.builder().put("readonly", true).build()).build(); @@ -1464,7 +1466,7 @@ private void mockBlobContainerForGlobalMetadata( ClusterMetadataManifest clusterMetadataManifest, Metadata metadata ) throws IOException { - String mockManifestFileName = "manifest__1__2__C__456__1"; + String mockManifestFileName = "manifest__1__2__C__456__2"; BlobMetadata blobMetadata = new PlainBlobMetadata(mockManifestFileName, 1); when( blobContainer.listBlobsByPrefixInSortedOrder( From 418d9fd5787806449b51733612c95389e056e25e Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 28 May 2024 16:07:50 +0530 Subject: [PATCH 12/23] Removing buffer logic Signed-off-by: Himshikha Gupta --- .../routingtable/IndexRoutingTableInput.java | 46 +++++ .../IndexRoutingTableInputStream.java | 173 ------------------ ....java => IndexRoutingTableInputTests.java} | 13 +- 3 files changed, 52 insertions(+), 180 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java rename server/src/test/java/org/opensearch/gateway/remote/routingtable/{IndexRoutingTableInputStreamTests.java => IndexRoutingTableInputTests.java} (85%) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java new file mode 100644 index 0000000000000..937de104fa5ad --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; + +import java.io.IOException; +import java.util.Iterator; + +public class IndexRoutingTableInput { + + private final IndexRoutingTableHeader indexRoutingTableHeader; + private final Iterator shardIter; + private int shardCount; + + public IndexRoutingTableInput(IndexRoutingTable indexRoutingTable) { + this.shardIter = indexRoutingTable.iterator(); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); + this.shardCount = indexRoutingTable.shards().size(); + } + + public BytesReference write() throws IOException { + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput); + indexRoutingTableHeader.write(out); + out.writeVInt(shardCount); + while (shardIter.hasNext()) { + IndexShardRoutingTable next = shardIter.next(); + IndexShardRoutingTable.Builder.writeTo(next, out); + } + out.writeLong(out.getChecksum()); + out.flush(); + return bytesStreamOutput.bytes(); + } + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java deleted file mode 100644 index d4e2594bf153c..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.routingtable; - -import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Iterator; - -public class IndexRoutingTableInputStream extends InputStream { - - /** - * The buffer where data is stored. - */ - protected byte[] buf; - - /** - * The number of valid bytes in the buffer. - */ - protected int count; - - /** - * The buffer left over from the last fill - */ - protected byte[] leftOverBuf; - - /** - * The mark position - */ - protected int markPos = -1; - - /** - * The read limit - */ - protected int markLimit; - - /** - * The position - */ - protected int pos; - - private static final int BUFFER_SIZE = 8192; - - private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; - private final BytesStreamOutput bytesStreamOutput; - private final BufferedChecksumStreamOutput out; - - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable) throws IOException { - this(indexRoutingTable, BUFFER_SIZE); - } - - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) throws IOException { - this.buf = new byte[size]; - this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); - this.bytesStreamOutput = new BytesStreamOutput(); - this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); - - initialFill(indexRoutingTable.shards().size()); - } - - @Override - public int read() throws IOException { - if (pos >= count) { - maybeResizeAndFill(); - if (pos >= count) return -1; - } - return buf[pos++] & 0xff; - } - - private void initialFill(int shardCount) throws IOException { - indexRoutingTableHeader.write(out); - out.writeVInt(shardCount); - - System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0, buf, 0, bytesStreamOutput.bytes().length()); - count = bytesStreamOutput.bytes().length(); - bytesStreamOutput.reset(); - fill(buf); - } - - private void fill(byte[] buf) throws IOException { - if (leftOverBuf != null) { - if (leftOverBuf.length > buf.length - count) { - // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in - // leftOverBuf. - System.arraycopy(leftOverBuf, 0, buf, count, buf.length - count); - byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; - System.arraycopy(leftOverBuf, buf.length - count, tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); - leftOverBuf = tempLeftOverBuffer; - count = buf.length - count; - } else { - System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); - count += leftOverBuf.length; - leftOverBuf = null; - } - } - - if (count < buf.length && shardIter.hasNext()) { - IndexShardRoutingTable next = shardIter.next(); - IndexShardRoutingTable.Builder.writeTo(next, out); - // Add checksum for the file after all shards are done - if (!shardIter.hasNext()) { - out.writeLong(out.getChecksum()); - } - out.flush(); - BytesReference bytesRef = bytesStreamOutput.bytes(); - bytesStreamOutput.reset(); - - if (bytesRef.length() < buf.length - count) { - System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, bytesRef.length()); - count += bytesRef.length(); - leftOverBuf = null; - } else { - System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); - leftOverBuf = new byte[bytesRef.length() - (buf.length - count)]; - System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count, leftOverBuf, 0, bytesRef.length() - (buf.length - count)); - count = buf.length; - } - } - } - - private void maybeResizeAndFill() throws IOException { - byte[] buffer = buf; - if (markPos == -1) pos = 0; /* no mark: throw away the buffer */ - else if (pos >= buffer.length) { /* no room left in buffer */ - if (markPos > 0) { /* can throw away early part of the buffer */ - int sz = pos - markPos; - System.arraycopy(buffer, markPos, buffer, 0, sz); - pos = sz; - markPos = 0; - } else if (buffer.length >= markLimit) { - markPos = -1; /* buffer got too big, invalidate mark */ - pos = 0; /* drop buffer contents */ - } else { /* grow buffer */ - int nsz = markLimit + 1; - byte[] nbuf = new byte[nsz]; - System.arraycopy(buffer, 0, nbuf, 0, pos); - buffer = nbuf; - } - } - count = pos; - fill(buffer); - } - - @Override - public void mark(int readlimit) { - markLimit = readlimit; - markPos = pos; - } - - @Override - public boolean markSupported() { - return true; - } - - @Override - public void reset() throws IOException { - if (markPos < 0) throw new IOException("Resetting to invalid mark"); - pos = markPos; - } -} diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java similarity index 85% rename from server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java rename to server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java index a6e2e4bda00e8..a37883ed7a5a5 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java @@ -17,12 +17,11 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.atomic.AtomicInteger; -public class IndexRoutingTableInputStreamTests extends OpenSearchTestCase { +public class IndexRoutingTableInputTests extends OpenSearchTestCase { - public void testRoutingTableInputStream() { + public void testRoutingTableInput() { Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) .build(); @@ -31,9 +30,9 @@ public void testRoutingTableInputStream() { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); + IndexRoutingTableInput indexRouting = new IndexRoutingTableInput(indexShardRoutingTables); - IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRouting.write().streamInput()); IndexRoutingTable indexRoutingTable = reader.readIndexRoutingTable(metadata.index("test").getIndex()); assertEquals(1, indexRoutingTable.getShards().size()); @@ -55,9 +54,9 @@ public void testRoutingTableInputStreamWithInvalidIndex() { AtomicInteger assertionError = new AtomicInteger(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); + IndexRoutingTableInput indexRouting = new IndexRoutingTableInput(indexShardRoutingTables); - IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRouting.write().streamInput()); reader.readIndexRoutingTable(metadata.index("invalid-index").getIndex()); } catch (AssertionError e) { From 7f33ba27989a7c5e4a7648644fdf2aa712324b2d Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 29 May 2024 13:07:16 +0530 Subject: [PATCH 13/23] Move BufferedChecksum streams to libs/core Signed-off-by: Himshikha Gupta --- .../core}/common/io/stream/BufferedChecksumStreamInput.java | 4 +--- .../core}/common/io/stream/BufferedChecksumStreamOutput.java | 2 +- .../gateway/remote/routingtable/IndexRoutingTableInput.java | 2 +- .../routingtable/IndexRoutingTableInputStreamReader.java | 2 +- .../org/opensearch/index/translog/BaseTranslogReader.java | 2 +- .../src/main/java/org/opensearch/index/translog/Translog.java | 4 ++-- .../java/org/opensearch/index/translog/TranslogHeader.java | 4 ++-- .../java/org/opensearch/index/translog/TranslogSnapshot.java | 2 +- .../java/org/opensearch/index/translog/TranslogWriter.java | 2 +- .../java/org/opensearch/snapshots/BlobStoreFormatTests.java | 2 +- 10 files changed, 12 insertions(+), 14 deletions(-) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/BufferedChecksumStreamInput.java (96%) rename {server/src/main/java/org/opensearch => libs/core/src/main/java/org/opensearch/core}/common/io/stream/BufferedChecksumStreamOutput.java (98%) diff --git a/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamInput.java similarity index 96% rename from server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamInput.java index f3341712275f9..41680961b36e9 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamInput.java @@ -30,12 +30,10 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.apache.lucene.util.BitUtil; -import org.opensearch.core.common.io.stream.FilterStreamInput; -import org.opensearch.core.common.io.stream.StreamInput; import java.io.EOFException; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java similarity index 98% rename from server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java rename to libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 254f228f1c739..0f370f2d2f1fe 100644 --- a/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.io.stream; +package org.opensearch.core.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java index 937de104fa5ad..bd143334b9f28 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java @@ -10,7 +10,7 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java index e2b4f5da5f6e9..d1c3d22a2cc34 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; diff --git a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java index 9088eb6b20fb8..37af1dcbeab8b 100644 --- a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java +++ b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java @@ -32,7 +32,7 @@ package org.opensearch.index.translog; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 18b10cb886996..01c1697d75e3b 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -38,8 +38,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index c622214a3a69d..66a9fe08d06b5 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -40,8 +40,8 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.Channels; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java index a6e322bf58fff..521472f4d64a0 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java @@ -32,7 +32,7 @@ package org.opensearch.index.translog; import org.opensearch.common.io.Channels; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.index.seqno.SequenceNumbers; import java.io.EOFException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index da3c7a8dee219..2176b9ce40687 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -42,7 +42,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Channels; import org.opensearch.common.io.DiskIoBufferPool; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 8d9a444c6f9e2..0f020a73cbf7f 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -46,7 +46,7 @@ import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.io.Streams; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; From 42efb38774ffa1f877254faf9515a922efda0567 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 29 May 2024 13:18:41 +0530 Subject: [PATCH 14/23] Spotless fix Signed-off-by: Himshikha Gupta --- .../gateway/remote/routingtable/IndexRoutingTableInput.java | 2 +- .../src/main/java/org/opensearch/index/translog/Translog.java | 4 ++-- .../java/org/opensearch/index/translog/TranslogWriter.java | 2 +- .../java/org/opensearch/snapshots/BlobStoreFormatTests.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java index bd143334b9f28..3790351e5f7fa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java @@ -10,9 +10,9 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import java.io.IOException; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 01c1697d75e3b..87e0c21b8203c 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -38,8 +38,6 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; @@ -50,6 +48,8 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.index.shard.ShardId; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index 2176b9ce40687..b0c7d51c3e43b 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -42,7 +42,6 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Channels; import org.opensearch.common.io.DiskIoBufferPool; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; @@ -51,6 +50,7 @@ import org.opensearch.core.Assertions; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 0f020a73cbf7f..95a8267734a07 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -46,10 +46,10 @@ import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.io.Streams; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.CompressorRegistry; import org.opensearch.core.xcontent.ToXContent; From 9b50f8a5226482752effe43ea402469c5ff0ddf6 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Wed, 29 May 2024 14:41:27 +0530 Subject: [PATCH 15/23] Refactor RemoteIndexRoutingTable read Signed-off-by: Arpit Bandejiya --- .../routingtable/IndexRoutingTableInput.java | 46 ------------------- ...ava => RemoteIndexRoutingTableObject.java} | 42 +++++++++++------ ...> RemoteIndexRoutingTableObjectTests.java} | 34 ++++++++------ 3 files changed, 49 insertions(+), 73 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java rename server/src/main/java/org/opensearch/gateway/remote/routingtable/{IndexRoutingTableInputStreamReader.java => RemoteIndexRoutingTableObject.java} (56%) rename server/src/test/java/org/opensearch/gateway/remote/routingtable/{IndexRoutingTableInputTests.java => RemoteIndexRoutingTableObjectTests.java} (62%) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java deleted file mode 100644 index 3790351e5f7fa..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInput.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.routingtable; - -import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; - -import java.io.IOException; -import java.util.Iterator; - -public class IndexRoutingTableInput { - - private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; - private int shardCount; - - public IndexRoutingTableInput(IndexRoutingTable indexRoutingTable) { - this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); - this.shardCount = indexRoutingTable.shards().size(); - } - - public BytesReference write() throws IOException { - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput); - indexRoutingTableHeader.write(out); - out.writeVInt(shardCount); - while (shardIter.hasNext()) { - IndexShardRoutingTable next = shardIter.next(); - IndexShardRoutingTable.Builder.writeTo(next, out); - } - out.writeLong(out.getChecksum()); - out.flush(); - return bytesStreamOutput.bytes(); - } - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java similarity index 56% rename from server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java rename to server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java index d1c3d22a2cc34..263fb29120553 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java @@ -8,44 +8,58 @@ package org.opensearch.gateway.remote.routingtable; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; -import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; +import java.util.Iterator; -public class IndexRoutingTableInputStreamReader { +public class RemoteIndexRoutingTableObject { - private final StreamInput streamInput; + private final IndexRoutingTableHeader indexRoutingTableHeader; + private final Iterator shardIter; + private final int shardCount; - private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStreamReader.class); + public RemoteIndexRoutingTableObject(IndexRoutingTable indexRoutingTable) { + this.shardIter = indexRoutingTable.iterator(); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); + this.shardCount = indexRoutingTable.shards().size(); + } - public IndexRoutingTableInputStreamReader(InputStream inputStream) throws IOException { - streamInput = new InputStreamStreamInput(inputStream); + public BytesReference write() throws IOException { + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput); + indexRoutingTableHeader.write(out); + out.writeVInt(shardCount); + while (shardIter.hasNext()) { + IndexShardRoutingTable next = shardIter.next(); + IndexShardRoutingTable.Builder.writeTo(next, out); + } + out.writeLong(out.getChecksum()); + out.flush(); + return bytesStreamOutput.bytes(); } - public IndexRoutingTable readIndexRoutingTable(Index index) throws IOException { + public static IndexRoutingTable read(InputStream inputStream, Index index) throws IOException { try { - try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(streamInput, "assertion")) { + try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) { // Read the Table Header first and confirm the index IndexRoutingTableHeader indexRoutingTableHeader = IndexRoutingTableHeader.read(in); assert indexRoutingTableHeader.getIndexName().equals(index.getName()); int numberOfShardRouting = in.readVInt(); - logger.debug("Number of Index Routing Table {}", numberOfShardRouting); IndexRoutingTable.Builder indicesRoutingTable = IndexRoutingTable.builder(index); for (int idx = 0; idx < numberOfShardRouting; idx++) { IndexShardRoutingTable indexShardRoutingTable = IndexShardRoutingTable.Builder.readFrom(in); - logger.debug("Index Shard Routing Table reading {}", indexShardRoutingTable); indicesRoutingTable.addIndexShard(indexShardRoutingTable); - } verifyCheckSum(in); return indicesRoutingTable.build(); @@ -55,7 +69,7 @@ public IndexRoutingTable readIndexRoutingTable(Index index) throws IOException { } } - private void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException { + public static void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException { long expectedChecksum = in.getChecksum(); long readChecksum = in.readLong(); if (readChecksum != expectedChecksum) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java similarity index 62% rename from server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java rename to server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java index a37883ed7a5a5..23b853f488f46 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java @@ -19,25 +19,36 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; -public class IndexRoutingTableInputTests extends OpenSearchTestCase { +public class RemoteIndexRoutingTableObjectTests extends OpenSearchTestCase { public void testRoutingTableInput() { + int numberOfShards = randomIntBetween(1, 10); + int numberOfReplicas = randomIntBetween(1, 10); Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .put( + IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(numberOfShards) + .numberOfReplicas(numberOfReplicas) + ) .build(); RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - IndexRoutingTableInput indexRouting = new IndexRoutingTableInput(indexShardRoutingTables); - - IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRouting.write().streamInput()); - IndexRoutingTable indexRoutingTable = reader.readIndexRoutingTable(metadata.index("test").getIndex()); + RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); + IndexRoutingTable indexRoutingTable = RemoteIndexRoutingTableObject.read( + indexRouting.write().streamInput(), + metadata.index("test").getIndex() + ); - assertEquals(1, indexRoutingTable.getShards().size()); + assertEquals(numberOfShards, indexRoutingTable.getShards().size()); assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); - assertEquals(indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), 2); + assertEquals( + indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), + numberOfShards * (1 + numberOfReplicas) + ); } catch (IOException e) { throw new RuntimeException(e); } @@ -54,11 +65,8 @@ public void testRoutingTableInputStreamWithInvalidIndex() { AtomicInteger assertionError = new AtomicInteger(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - IndexRoutingTableInput indexRouting = new IndexRoutingTableInput(indexShardRoutingTables); - - IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRouting.write().streamInput()); - reader.readIndexRoutingTable(metadata.index("invalid-index").getIndex()); - + RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); + RemoteIndexRoutingTableObject.read(indexRouting.write().streamInput(), metadata.index("invalid-index").getIndex()); } catch (AssertionError e) { assertionError.getAndIncrement(); } catch (IOException e) { From a7f7cab814387e39e44b08055e58c6addd08b53c Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Wed, 29 May 2024 15:42:15 +0530 Subject: [PATCH 16/23] Add Manifest Tests and spotless fix Signed-off-by: Arpit Bandejiya --- .../stream/BufferedChecksumStreamOutput.java | 1 - .../remote/ClusterMetadataManifestTests.java | 83 ++++++++++++++++++- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 0f370f2d2f1fe..422f956c0cd47 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -34,7 +34,6 @@ import org.apache.lucene.store.BufferedChecksum; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.zip.CRC32; diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index e5fee8a5d7ad9..d1f559eb75f85 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -99,7 +99,7 @@ public void testClusterMetadataManifestXContent() throws IOException { Version.CURRENT, "test-node-id", false, - ClusterMetadataManifest.CODEC_V2, + ClusterMetadataManifest.CODEC_V3, null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", @@ -123,7 +123,9 @@ public void testClusterMetadataManifestXContent() throws IOException { "custom--weighted_routing_netadata-file" ) ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), + 1L, + randomUploadedIndexMetadataList() ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -169,7 +171,9 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "custom--weighted_routing_netadata-file" ) ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), + 1L, + randomUploadedIndexMetadataList() ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -311,6 +315,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { public void testClusterMetadataManifestXContentV2() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( 1L, 1L, @@ -320,10 +325,80 @@ public void testClusterMetadataManifestXContentV2() throws IOException { "test-node-id", false, ClusterMetadataManifest.CODEC_V2, - "test-metadata", + null, + Collections.singletonList(uploadedIndexMetadata), + "prev-cluster-uuid", + true, + uploadedMetadataAttribute, + uploadedMetadataAttribute, + uploadedMetadataAttribute, + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) + ) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), + 0, + new ArrayList<>() + ); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV2(parser); + assertEquals(originalManifest, fromXContentManifest); + } + } + + public void testClusterMetadataManifestXContentV3() throws IOException { + UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); + ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( + 1L, + 1L, + "test-cluster-uuid", + "test-state-uuid", + Version.CURRENT, + "test-node-id", + false, + ClusterMetadataManifest.CODEC_V3, + null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", true, + uploadedMetadataAttribute, + uploadedMetadataAttribute, + uploadedMetadataAttribute, + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) + ) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), 1L, Collections.singletonList(uploadedIndexMetadata) ); From 71e151f6570e73cdd1d2d5a68a3ee25dedaaf264 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Thu, 30 May 2024 12:38:35 +0530 Subject: [PATCH 17/23] Fix remoteClusterServiceTests Signed-off-by: Arpit Bandejiya --- .../gateway/remote/RemoteClusterStateService.java | 2 +- .../gateway/remote/RemoteClusterStateServiceTests.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 9ce7d67a5706b..9e9caf47a0997 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -215,7 +215,7 @@ public class RemoteClusterStateService implements Closeable { + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + "updated : [{}], custom metadata updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2; // ToXContent Params with gateway mode. diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index d95d95fba630f..e64865da69b65 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -547,7 +547,7 @@ private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMe ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() - .codecVersion(2) + .codecVersion(3) .indices(Collections.emptyList()) .clusterTerm(1L) .stateVersion(1L) @@ -1559,7 +1559,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap1 = Map.of("index-uuid1", indexMetadata1, "index-uuid2", indexMetadata2); mockBlobContainerForGlobalMetadata(blobContainer1, clusterManifest1, metadata1); - mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V2); + mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V3); List uploadedIndexMetadataList2 = List.of( new UploadedIndexMetadata("index1", "index-uuid1", "key1"), @@ -1591,7 +1591,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap2 = Map.of("index-uuid1", indexMetadata3, "index-uuid2", indexMetadata4); mockBlobContainerForGlobalMetadata(blobContainer2, clusterManifest2, metadata2); - mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V2); + mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V3); // differGlobalMetadata controls which one of IndexMetadata or Metadata object would be different // when comparing cluster-uuid3 and cluster-uuid1 state. @@ -1625,7 +1625,7 @@ private void mockObjectsForGettingPreviousClusterUUID( clusterUUIDCommitted.getOrDefault("cluster-uuid3", true) ); mockBlobContainerForGlobalMetadata(blobContainer3, clusterManifest3, metadata3); - mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V2); + mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V3); ArrayList mockBlobContainerOrderedList = new ArrayList<>( List.of(blobContainer1, blobContainer1, blobContainer3, blobContainer3, blobContainer2, blobContainer2) From a9ab9c10783656ff5bb4ecfa155f16c36319e6ed Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 31 May 2024 15:17:49 +0530 Subject: [PATCH 18/23] Remote store objects to implement writeable interface Signed-off-by: Himshikha Gupta --- .../routingtable/IndexRoutingTableHeader.java | 32 +++++----- .../RemoteIndexRoutingTableObject.java | 60 ++++++++++--------- .../IndexRoutingTableHeaderTests.java | 4 +- .../RemoteIndexRoutingTableObjectTests.java | 16 +++-- 4 files changed, 63 insertions(+), 49 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index bc99fea9b5c09..059b385394cb5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -16,6 +16,7 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.EOFException; import java.io.IOException; @@ -23,7 +24,7 @@ /** * The stored header information for the individual index routing table */ -public class IndexRoutingTableHeader { +public class IndexRoutingTableHeader implements Writeable { public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; public static final int INITIAL_VERSION = 1; @@ -35,41 +36,42 @@ public IndexRoutingTableHeader(String indexName) { } /** - * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} + * Reads the contents on the stream into the corresponding {@link IndexRoutingTableHeader} * * @param in - * @return IndexRoutingTableHeader * @throws IOException */ - public static IndexRoutingTableHeader read(StreamInput in) throws IOException { + public IndexRoutingTableHeader(StreamInput in) throws IOException { try { readHeaderVersion(in); - final String name = in.readString(); - return new IndexRoutingTableHeader(name); + indexName = in.readString(); } catch (EOFException e) { throw new IOException("index routing header truncated", e); } } - static int readHeaderVersion(final StreamInput in) throws IOException { - final int version; + private void readHeaderVersion(final StreamInput in) throws IOException { try { - version = CodecUtil.checkHeader(new InputStreamDataInput(in), INDEX_ROUTING_HEADER_CODEC, INITIAL_VERSION, CURRENT_VERSION); + CodecUtil.checkHeader(new InputStreamDataInput(in), INDEX_ROUTING_HEADER_CODEC, INITIAL_VERSION, CURRENT_VERSION); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { throw new IOException("index routing table header corrupted", e); } - return version; } /** - * Returns the bytes reference for the {@link IndexRoutingTableHeader} + * Write the IndexRoutingTable to given stream. * + * @param out stream to write * @throws IOException */ - public void write(StreamOutput out) throws IOException { - CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); - out.writeString(indexName); - out.flush(); + public void writeTo(StreamOutput out) throws IOException { + try { + CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); + out.writeString(indexName); + out.flush(); + } catch (IOException e) { + throw new IOException("Failed to write IndexRoutingTable header", e); + } } public String getIndexName() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java index 263fb29120553..91ab7ecf413c0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java @@ -10,49 +10,33 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.Index; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; -import java.util.Iterator; -public class RemoteIndexRoutingTableObject { +/** + * Remote store object for IndexRoutingTable + */ +public class RemoteIndexRoutingTableObject implements Writeable { - private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; - private final int shardCount; + private final IndexRoutingTable indexRoutingTable; public RemoteIndexRoutingTableObject(IndexRoutingTable indexRoutingTable) { - this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); - this.shardCount = indexRoutingTable.shards().size(); - } - - public BytesReference write() throws IOException { - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput); - indexRoutingTableHeader.write(out); - out.writeVInt(shardCount); - while (shardIter.hasNext()) { - IndexShardRoutingTable next = shardIter.next(); - IndexShardRoutingTable.Builder.writeTo(next, out); - } - out.writeLong(out.getChecksum()); - out.flush(); - return bytesStreamOutput.bytes(); + this.indexRoutingTable = indexRoutingTable; } - public static IndexRoutingTable read(InputStream inputStream, Index index) throws IOException { + public RemoteIndexRoutingTableObject(InputStream inputStream, Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) { // Read the Table Header first and confirm the index - IndexRoutingTableHeader indexRoutingTableHeader = IndexRoutingTableHeader.read(in); + IndexRoutingTableHeader indexRoutingTableHeader = new IndexRoutingTableHeader(in); assert indexRoutingTableHeader.getIndexName().equals(index.getName()); int numberOfShardRouting = in.readVInt(); @@ -62,14 +46,34 @@ public static IndexRoutingTable read(InputStream inputStream, Index index) throw indicesRoutingTable.addIndexShard(indexShardRoutingTable); } verifyCheckSum(in); - return indicesRoutingTable.build(); + indexRoutingTable = indicesRoutingTable.build(); } } catch (EOFException e) { throw new IOException("Indices Routing table is corrupted", e); } } - public static void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException { + public IndexRoutingTable getIndexRoutingTable() { + return indexRoutingTable; + } + + public void writeTo(StreamOutput streamOutput) throws IOException { + try { + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(streamOutput); + IndexRoutingTableHeader indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); + indexRoutingTableHeader.writeTo(out); + out.writeVInt(indexRoutingTable.shards().size()); + for (IndexShardRoutingTable next : indexRoutingTable) { + IndexShardRoutingTable.Builder.writeTo(next, out); + } + out.writeLong(out.getChecksum()); + out.flush(); + } catch (IOException e) { + throw new IOException("Failed to write IndexRoutingTable to stream", e); + } + } + + private void verifyCheckSum(BufferedChecksumStreamInput in) throws IOException { long expectedChecksum = in.getChecksum(); long readChecksum = in.readLong(); if (readChecksum != expectedChecksum) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index 0f42508615a26..187ef2b928f75 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -19,10 +19,10 @@ public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { public void testIndexRoutingTableHeader() throws IOException { IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); BytesStreamOutput out = new BytesStreamOutput(); - header.write(out); + header.writeTo(out); BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); - IndexRoutingTableHeader headerRead = IndexRoutingTableHeader.read(in); + IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in); assertEquals("dummyIndex", headerRead.getIndexName()); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java index 23b853f488f46..d4d01ea04b281 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java @@ -14,6 +14,7 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -38,11 +39,13 @@ public void testRoutingTableInput() { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); - IndexRoutingTable indexRoutingTable = RemoteIndexRoutingTableObject.read( - indexRouting.write().streamInput(), + BytesStreamOutput streamOutput = new BytesStreamOutput(); + indexRouting.writeTo(streamOutput); + RemoteIndexRoutingTableObject remoteIndexRoutingTable = new RemoteIndexRoutingTableObject( + streamOutput.bytes().streamInput(), metadata.index("test").getIndex() ); - + IndexRoutingTable indexRoutingTable = remoteIndexRoutingTable.getIndexRoutingTable(); assertEquals(numberOfShards, indexRoutingTable.getShards().size()); assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); assertEquals( @@ -66,7 +69,12 @@ public void testRoutingTableInputStreamWithInvalidIndex() { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); - RemoteIndexRoutingTableObject.read(indexRouting.write().streamInput(), metadata.index("invalid-index").getIndex()); + BytesStreamOutput streamOutput = new BytesStreamOutput(); + indexRouting.writeTo(streamOutput); + RemoteIndexRoutingTableObject remoteIndexRoutingTable = new RemoteIndexRoutingTableObject( + streamOutput.bytes().streamInput(), + metadata.index("invalid-index").getIndex() + ); } catch (AssertionError e) { assertionError.getAndIncrement(); } catch (IOException e) { From 4167eebf27f4d69fbf350ea06aa6eb0eadb6c851 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 3 Jun 2024 15:19:52 +0530 Subject: [PATCH 19/23] addressing pr comments Signed-off-by: Himshikha Gupta --- ...bleObject.java => RemoteIndexRoutingTable.java} | 7 ++++--- .../routingtable/IndexRoutingTableHeaderTests.java | 12 +++++++----- .../RemoteIndexRoutingTableObjectTests.java | 14 ++++++-------- 3 files changed, 17 insertions(+), 16 deletions(-) rename server/src/main/java/org/opensearch/gateway/remote/routingtable/{RemoteIndexRoutingTableObject.java => RemoteIndexRoutingTable.java} (93%) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java similarity index 93% rename from server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java rename to server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java index 91ab7ecf413c0..5641ef01df8d2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java @@ -24,15 +24,15 @@ /** * Remote store object for IndexRoutingTable */ -public class RemoteIndexRoutingTableObject implements Writeable { +public class RemoteIndexRoutingTable implements Writeable { private final IndexRoutingTable indexRoutingTable; - public RemoteIndexRoutingTableObject(IndexRoutingTable indexRoutingTable) { + public RemoteIndexRoutingTable(IndexRoutingTable indexRoutingTable) { this.indexRoutingTable = indexRoutingTable; } - public RemoteIndexRoutingTableObject(InputStream inputStream, Index index) throws IOException { + public RemoteIndexRoutingTable(InputStream inputStream, Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) { // Read the Table Header first and confirm the index @@ -57,6 +57,7 @@ public IndexRoutingTable getIndexRoutingTable() { return indexRoutingTable; } + @Override public void writeTo(StreamOutput streamOutput) throws IOException { try { BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(streamOutput); diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index 187ef2b928f75..bea65936fc32e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -18,12 +18,14 @@ public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { public void testIndexRoutingTableHeader() throws IOException { IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); - BytesStreamOutput out = new BytesStreamOutput(); - header.writeTo(out); + try (BytesStreamOutput out = new BytesStreamOutput()) { + header.writeTo(out); - BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); - IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in); - assertEquals("dummyIndex", headerRead.getIndexName()); + BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); + IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in); + assertEquals("dummyIndex", headerRead.getIndexName()); + + } } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java index d4d01ea04b281..d6fed51f692a5 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java @@ -37,11 +37,10 @@ public void testRoutingTableInput() { RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { - try { - RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); - BytesStreamOutput streamOutput = new BytesStreamOutput(); + RemoteIndexRoutingTable indexRouting = new RemoteIndexRoutingTable(indexShardRoutingTables); + try (BytesStreamOutput streamOutput = new BytesStreamOutput();) { indexRouting.writeTo(streamOutput); - RemoteIndexRoutingTableObject remoteIndexRoutingTable = new RemoteIndexRoutingTableObject( + RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable( streamOutput.bytes().streamInput(), metadata.index("test").getIndex() ); @@ -67,11 +66,10 @@ public void testRoutingTableInputStreamWithInvalidIndex() { RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); AtomicInteger assertionError = new AtomicInteger(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { - try { - RemoteIndexRoutingTableObject indexRouting = new RemoteIndexRoutingTableObject(indexShardRoutingTables); - BytesStreamOutput streamOutput = new BytesStreamOutput(); + RemoteIndexRoutingTable indexRouting = new RemoteIndexRoutingTable(indexShardRoutingTables); + try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { indexRouting.writeTo(streamOutput); - RemoteIndexRoutingTableObject remoteIndexRoutingTable = new RemoteIndexRoutingTableObject( + RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable( streamOutput.bytes().streamInput(), metadata.index("invalid-index").getIndex() ); From b3c18d876e86da5a8423c5f3b323375771424e64 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 3 Jun 2024 18:11:16 +0530 Subject: [PATCH 20/23] test fixes Signed-off-by: Himshikha Gupta --- .../IndexRoutingTableHeaderTests.java | 5 +++-- ...sts.java => RemoteIndexRoutingTableTests.java} | 15 ++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) rename server/src/test/java/org/opensearch/gateway/remote/routingtable/{RemoteIndexRoutingTableObjectTests.java => RemoteIndexRoutingTableTests.java} (87%) diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index bea65936fc32e..a3f0ac36a40f1 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -17,13 +17,14 @@ public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { public void testIndexRoutingTableHeader() throws IOException { - IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + IndexRoutingTableHeader header = new IndexRoutingTableHeader(indexName); try (BytesStreamOutput out = new BytesStreamOutput()) { header.writeTo(out); BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); IndexRoutingTableHeader headerRead = new IndexRoutingTableHeader(in); - assertEquals("dummyIndex", headerRead.getIndexName()); + assertEquals(indexName, headerRead.getIndexName()); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java similarity index 87% rename from server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java rename to server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java index d6fed51f692a5..72066d8afb45b 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableObjectTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableTests.java @@ -20,21 +20,22 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; -public class RemoteIndexRoutingTableObjectTests extends OpenSearchTestCase { +public class RemoteIndexRoutingTableTests extends OpenSearchTestCase { public void testRoutingTableInput() { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); int numberOfShards = randomIntBetween(1, 10); int numberOfReplicas = randomIntBetween(1, 10); Metadata metadata = Metadata.builder() .put( - IndexMetadata.builder("test") + IndexMetadata.builder(indexName) .settings(settings(Version.CURRENT)) .numberOfShards(numberOfShards) .numberOfReplicas(numberOfReplicas) ) .build(); - RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); + RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build(); initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { RemoteIndexRoutingTable indexRouting = new RemoteIndexRoutingTable(indexShardRoutingTables); @@ -42,14 +43,14 @@ public void testRoutingTableInput() { indexRouting.writeTo(streamOutput); RemoteIndexRoutingTable remoteIndexRoutingTable = new RemoteIndexRoutingTable( streamOutput.bytes().streamInput(), - metadata.index("test").getIndex() + metadata.index(indexName).getIndex() ); IndexRoutingTable indexRoutingTable = remoteIndexRoutingTable.getIndexRoutingTable(); assertEquals(numberOfShards, indexRoutingTable.getShards().size()); - assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); + assertEquals(metadata.index(indexName).getIndex(), indexRoutingTable.getIndex()); assertEquals( - indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), - numberOfShards * (1 + numberOfReplicas) + numberOfShards * (1 + numberOfReplicas), + indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size() ); } catch (IOException e) { throw new RuntimeException(e); From bbebce390c5edb4afb216d64b688e1407e44315d Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 4 Jun 2024 16:17:55 +0530 Subject: [PATCH 21/23] addressing PR comments Signed-off-by: Himshikha Gupta --- .../gateway/remote/ClusterMetadataManifest.java | 2 +- .../gateway/remote/RemoteClusterStateService.java | 1 + .../remote/routingtable/RemoteIndexRoutingTable.java | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 72e3826a3223a..b3b1bf37f8696 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -41,7 +41,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. - public static final int CODEC_V3 = 3; // In Codec V2 we introduce index routing-metadata in manifest file. + public static final int CODEC_V3 = 3; // In Codec V3, we introduce index routing-metadata in manifest file. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 9e9caf47a0997..9a0db3a2d25fd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -797,6 +797,7 @@ private ClusterMetadataManifest uploadManifest( uploadedTemplatesMetadata, uploadedCustomMetadataMap, clusterState.routingTable().version(), + // TODO: Add actual list of changed indices routing with index routing upload flow. new ArrayList<>() ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java index 5641ef01df8d2..c584e8400939f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java @@ -32,6 +32,12 @@ public RemoteIndexRoutingTable(IndexRoutingTable indexRoutingTable) { this.indexRoutingTable = indexRoutingTable; } + /** + * Reads data from inputStream and creates RemoteIndexRoutingTable object with the {@link IndexRoutingTable} + * @param inputStream + * @param index + * @throws IOException + */ public RemoteIndexRoutingTable(InputStream inputStream, Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(inputStream), "assertion")) { @@ -57,6 +63,11 @@ public IndexRoutingTable getIndexRoutingTable() { return indexRoutingTable; } + /** + * Writes {@link IndexRoutingTable} to the given stream + * @param streamOutput + * @throws IOException + */ @Override public void writeTo(StreamOutput streamOutput) throws IOException { try { From 24fe0636b5a15a997e68575c1742d32222610833 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 6 Jun 2024 11:49:36 +0530 Subject: [PATCH 22/23] Add package-info Signed-off-by: Himshikha Gupta --- .../gateway/remote/routingtable/package-info.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/package-info.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/package-info.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/package-info.java new file mode 100644 index 0000000000000..a6cb2251a5dd7 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Package containing class to perform operations on remote routing table. + */ +package org.opensearch.gateway.remote.routingtable; From 01019da4ed4b736eac2ff1802a056b82df9ad385 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 6 Jun 2024 12:13:06 +0530 Subject: [PATCH 23/23] Fixing javadoc failure Signed-off-by: Himshikha Gupta --- .../remote/routingtable/IndexRoutingTableHeader.java | 6 +++--- .../remote/routingtable/RemoteIndexRoutingTable.java | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 059b385394cb5..5baea6adba0c7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -38,8 +38,8 @@ public IndexRoutingTableHeader(String indexName) { /** * Reads the contents on the stream into the corresponding {@link IndexRoutingTableHeader} * - * @param in - * @throws IOException + * @param in streamInput + * @throws IOException exception thrown on failing to read from stream. */ public IndexRoutingTableHeader(StreamInput in) throws IOException { try { @@ -62,7 +62,7 @@ private void readHeaderVersion(final StreamInput in) throws IOException { * Write the IndexRoutingTable to given stream. * * @param out stream to write - * @throws IOException + * @throws IOException exception thrown on failing to write to stream. */ public void writeTo(StreamOutput out) throws IOException { try { diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java index c584e8400939f..17c55190da07f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTable.java @@ -34,9 +34,9 @@ public RemoteIndexRoutingTable(IndexRoutingTable indexRoutingTable) { /** * Reads data from inputStream and creates RemoteIndexRoutingTable object with the {@link IndexRoutingTable} - * @param inputStream - * @param index - * @throws IOException + * @param inputStream input stream with index routing data + * @param index index for the current routing data + * @throws IOException exception thrown on failing to read from stream. */ public RemoteIndexRoutingTable(InputStream inputStream, Index index) throws IOException { try { @@ -65,8 +65,8 @@ public IndexRoutingTable getIndexRoutingTable() { /** * Writes {@link IndexRoutingTable} to the given stream - * @param streamOutput - * @throws IOException + * @param streamOutput output stream to write + * @throws IOException exception thrown on failing to write to stream. */ @Override public void writeTo(StreamOutput streamOutput) throws IOException {