Skip to content

Commit

Permalink
Upload consistent settings and customs object to remote
Browse files Browse the repository at this point in the history 
Signed-off-by: Shivansh Arora <[email protected]>
  • Loading branch information
@shiv0408
shiv0408 committed May 31, 2024
1 parent e8fec03 commit 00f0249
Show file tree
Hide file tree
Showing 18 changed files with 652 additions and 91 deletions.
29 changes: 29 additions & 0 deletions server/src/main/java/org/opensearch/cluster/ClusterModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,35 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
DecommissionAttributeMetadata::fromXContent
)
);
// Cluster State
entries.add(
new NamedXContentRegistry.Entry(
ClusterState.Custom.class,
new ParseField(SnapshotsInProgress.TYPE),
SnapshotsInProgress::fromXContent
)
);
entries.add(
new NamedXContentRegistry.Entry(
ClusterState.Custom.class,
new ParseField(RestoreInProgress.TYPE),
RestoreInProgress::fromXContent
)
);
entries.add(
new NamedXContentRegistry.Entry(
ClusterState.Custom.class,
new ParseField(SnapshotDeletionsInProgress.TYPE),
SnapshotDeletionsInProgress::fromXContent
)
);
entries.add(
new NamedXContentRegistry.Entry(
ClusterState.Custom.class,
new ParseField(RepositoryCleanupInProgress.TYPE),
RepositoryCleanupInProgress::fromXContent
)
);
return entries;
}

Expand Down
4 changes: 4 additions & 0 deletions server/src/main/java/org/opensearch/cluster/ClusterState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import org.opensearch.core.common.io.stream.VersionedNamedWriteable;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.discovery.Discovery;

import java.io.IOException;
Expand Down Expand Up @@ -154,6 +155,9 @@ default boolean isPrivate() {
return false;
}

static Custom fromXContent(XContentParser parser, String name) throws IOException {
return parser.namedObject(Custom.class, name, null);
}
}

private static final NamedDiffableValueSerializer<Custom> CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class);
Expand Down
10 changes: 5 additions & 5 deletions server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public static DiffableStringMap readFrom(StreamInput in) throws IOException {
return map.isEmpty() ? EMPTY : new DiffableStringMap(map);
}

DiffableStringMap(final Map<String, String> map) {
public DiffableStringMap(final Map<String, String> map) {
this.innerMap = Collections.unmodifiableMap(map);
}

Expand All @@ -100,7 +100,7 @@ public static Diff<DiffableStringMap> readDiffFrom(StreamInput in) throws IOExce

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startObject("diffable_string_map");
builder.field("inner_map", innerMap);
builder.endObject();
return builder;
Expand All @@ -110,9 +110,9 @@ public static DiffableStringMap fromXContent(XContentParser parser) throws IOExc
if (parser.currentToken() == null) { // fresh parser? move to next token
parser.nextToken();
}
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
parser.nextToken();
}
ensureFieldName(parser, parser.currentToken(), "diffable_string_map");
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
parser.nextToken();
ensureFieldName(parser, parser.currentToken(), "inner_map");
parser.nextToken();
Map<String, Object> innerMap = parser.mapOrdered();
Expand Down
5 changes: 5 additions & 0 deletions server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -986,6 +986,11 @@ public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata meta
return metadata1.templates.equals(metadata2.templates);
}

public static boolean isHashesOfConsistentSettingsEqual(Metadata metadata1, Metadata metadata2) {
return metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings);
}


public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) {
int customCount1 = 0;
for (Map.Entry<String, Custom> cursor : metadata1.customs.entrySet()) {
Expand Down
101 changes: 92 additions & 9 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata");
private static final ParseField UPLOADED_DISCOVERY_NODES_METADATA = new ParseField("uploaded_discovery_nodes_metadata");
private static final ParseField UPLOADED_CLUSTER_BLOCKS_METADATA = new ParseField("uploaded_cluster_blocks_metadata");
private static final ParseField UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA = new ParseField("uploaded_hashes_of_consistent_settings_metadata");
private static final ParseField UPLOADED_CLUSTER_STATE_CUSTOM_METADATA = new ParseField("uploaded_cluster_state_custom_metadata");
private static final ParseField DIFF_MANIFEST = new ParseField("diff_manifest");
private static final ParseField ROUTING_TABLE_VERSION_FIELD = new ParseField("routing_table_version");
private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing");
Expand Down Expand Up @@ -104,7 +106,9 @@ private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields
.routingTableVersion(routingTableVersion(fields))
.indicesRouting(indicesRouting(fields))
.metadataVersion(metadataVersion(fields))
.transientSettingsMetadata(transientSettingsMetadata(fields));
.transientSettingsMetadata(transientSettingsMetadata(fields))
.hashesOfConsistentSettings(hashesOfConsistentSettings(fields))
.clusterStateCustomMetadataMap(clusterStateCustomMetadata(fields));
}

private static long term(Object[] fields) {
Expand Down Expand Up @@ -200,6 +204,15 @@ private static UploadedMetadataAttribute transientSettingsMetadata(Object[] fiel
return (UploadedMetadataAttribute) fields[21];
}

private static UploadedMetadataAttribute hashesOfConsistentSettings(Object[] fields) {
return (UploadedMetadataAttribute) fields[22];
}

private static Map<String, UploadedMetadataAttribute> clusterStateCustomMetadata(Object[] fields) {
List<UploadedMetadataAttribute> customs = (List<UploadedMetadataAttribute>) fields[23];
return customs.stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity()));
}

private static final ConstructingObjectParser<ClusterMetadataManifest, Void> PARSER_V0 = new ConstructingObjectParser<>(
"cluster_metadata_manifest",
fields -> manifestV0Builder(fields).build()
Expand Down Expand Up @@ -299,6 +312,16 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
UploadedMetadataAttribute.PARSER,
UPLOADED_TRANSIENT_SETTINGS_METADATA
);
parser.declareNamedObject(
ConstructingObjectParser.optionalConstructorArg(),
UploadedMetadataAttribute.PARSER,
UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA
);
parser.declareNamedObjects(
ConstructingObjectParser.optionalConstructorArg(),
UploadedMetadataAttribute.PARSER,
UPLOADED_CLUSTER_STATE_CUSTOM_METADATA
);
}
}

Expand All @@ -325,6 +348,8 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
private final ClusterStateDiffManifest diffManifest;
private final long routingTableVersion;
private final List<UploadedIndexMetadata> indicesRouting;
private final UploadedMetadataAttribute uploadedHashesOfConsistentSettings;
private final Map<String, UploadedMetadataAttribute> uploadedClusterStateCustomMap;

public List<UploadedIndexMetadata> getIndices() {
return indices;
Expand Down Expand Up @@ -412,13 +437,26 @@ public Map<String, UploadedMetadataAttribute> getCustomMetadataMap() {
return uploadedCustomMetadataMap;
}

public long getRoutingTableVersion() {
return routingTableVersion;
}
public UploadedMetadataAttribute getDiscoverNodeMetadata() {
return uploadedDiscoveryNodesMetadata;
}

public Map<String, UploadedMetadataAttribute> getClusterStateCustomMap() {
return uploadedClusterStateCustomMap;
}

public UploadedMetadataAttribute getHashesOfConsistentSettings() {
return uploadedHashesOfConsistentSettings;
}

public long getRoutingTableVersion() {
return routingTableVersion;
}

public List<UploadedIndexMetadata> getIndicesRouting() {
return indicesRouting;
}

public List<UploadedIndexMetadata> getIndicesRouting() {
return indicesRouting;
}
public boolean hasMetadataAttributesFiles() {
return uploadedCoordinationMetadata != null
|| uploadedSettingsMetadata != null
Expand Down Expand Up @@ -449,7 +487,9 @@ public ClusterMetadataManifest(
long routingTableVersion,
List<UploadedIndexMetadata> indicesRouting,
long metadataVersion,
UploadedMetadataAttribute uploadedTransientSettingsMetadata
UploadedMetadataAttribute uploadedTransientSettingsMetadata,
UploadedMetadataAttribute uploadedHashesOfConsistentSettings,
Map<String, UploadedMetadataAttribute> uploadedClusterStateCustomMap
) {
this.clusterTerm = clusterTerm;
this.stateVersion = version;
Expand All @@ -476,6 +516,10 @@ public ClusterMetadataManifest(
this.indicesRouting = Collections.unmodifiableList(indicesRouting);
this.metadataVersion = metadataVersion;
this.uploadedTransientSettingsMetadata = uploadedTransientSettingsMetadata;
this.uploadedHashesOfConsistentSettings = uploadedHashesOfConsistentSettings;
this.uploadedClusterStateCustomMap = Collections.unmodifiableMap(
uploadedClusterStateCustomMap != null ? uploadedClusterStateCustomMap : new HashMap<>()
);
}

public ClusterMetadataManifest(StreamInput in) throws IOException {
Expand Down Expand Up @@ -507,6 +551,10 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new));
this.metadataVersion = in.readLong();
this.uploadedTransientSettingsMetadata = new UploadedMetadataAttribute(in);
this.uploadedHashesOfConsistentSettings = new UploadedMetadataAttribute(in);
this.uploadedClusterStateCustomMap = Collections.unmodifiableMap(
in.readMap(StreamInput::readString, UploadedMetadataAttribute::new)
);
} else if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
this.codecVersion = in.readInt();
this.globalMetadataFileName = in.readString();
Expand All @@ -521,6 +569,8 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
this.diffManifest = null;
this.metadataVersion = -1;
this.uploadedTransientSettingsMetadata = null;
this.uploadedHashesOfConsistentSettings = null;
this.uploadedClusterStateCustomMap = null;
} else {
this.codecVersion = CODEC_V0; // Default codec
this.globalMetadataFileName = null;
Expand All @@ -535,6 +585,8 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
this.diffManifest = null;
this.metadataVersion = -1;
this.uploadedTransientSettingsMetadata = null;
this.uploadedHashesOfConsistentSettings = null;
this.uploadedClusterStateCustomMap = null;
}
}

Expand Down Expand Up @@ -625,6 +677,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
}
builder.endArray();
builder.startObject(UPLOADED_CLUSTER_STATE_CUSTOM_METADATA.getPreferredName());
for (UploadedMetadataAttribute attribute : getClusterStateCustomMap().values()) {
attribute.toXContent(builder, params);
}
builder.endObject();
if (getHashesOfConsistentSettings() != null) {
builder.startObject(UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA.getPreferredName());
getHashesOfConsistentSettings().toXContent(builder, params);
builder.endObject();
}
}
return builder;
}
Expand All @@ -651,6 +713,8 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeCollection(indicesRouting);
out.writeLong(metadataVersion);
uploadedTransientSettingsMetadata.writeTo(out);
uploadedHashesOfConsistentSettings.writeTo(out);
out.writeMap(uploadedClusterStateCustomMap, StreamOutput::writeString, (o, v) -> v.writeTo(o));
} else if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
out.writeInt(codecVersion);
out.writeString(globalMetadataFileName);
Expand All @@ -659,6 +723,7 @@ public void writeTo(StreamOutput out) throws IOException {

@Override
public boolean equals(Object o) {
// ToDo: update this method to contain new attributes
if (this == o) {
return true;
}
Expand All @@ -684,6 +749,7 @@ public boolean equals(Object o) {

@Override
public int hashCode() {
// ToDo: update this method to contain new attributes
return Objects.hash(
codecVersion,
globalMetadataFileName,
Expand Down Expand Up @@ -753,6 +819,8 @@ public static class Builder {
private ClusterStateDiffManifest diffManifest;
private long routingTableVersion;
private List<UploadedIndexMetadata> indicesRouting;
private UploadedMetadataAttribute hashesOfConsistentSettings;
private Map<String, UploadedMetadataAttribute> clusterStateCustomMetadataMap;

public Builder indices(List<UploadedIndexMetadata> indices) {
this.indices = indices;
Expand Down Expand Up @@ -882,10 +950,21 @@ public Builder diffManifest(ClusterStateDiffManifest diffManifest) {
return this;
}

public Builder hashesOfConsistentSettings(UploadedMetadataAttribute hashesOfConsistentSettings) {
this.hashesOfConsistentSettings = hashesOfConsistentSettings;
return this;
}

public Builder clusterStateCustomMetadataMap(Map<String, UploadedMetadataAttribute> clusterStateCustomMetadataMap) {
this.clusterStateCustomMetadataMap = clusterStateCustomMetadataMap;
return this;
}

public Builder() {
indices = new ArrayList<>();
customMetadataMap = new HashMap<>();
indicesRouting = new ArrayList<>();
clusterStateCustomMetadataMap = new HashMap<>();
}

public Builder(ClusterMetadataManifest manifest) {
Expand All @@ -908,6 +987,8 @@ public Builder(ClusterMetadataManifest manifest) {
this.diffManifest = manifest.diffManifest;
this.routingTableVersion = manifest.routingTableVersion;
this.indicesRouting = new ArrayList<>(manifest.indicesRouting);
this.hashesOfConsistentSettings = manifest.uploadedHashesOfConsistentSettings;
this.clusterStateCustomMetadataMap = manifest.uploadedClusterStateCustomMap;
}

public ClusterMetadataManifest build() {
Expand All @@ -934,7 +1015,9 @@ public ClusterMetadataManifest build() {
routingTableVersion,
indicesRouting,
metadataVersion,
transientSettingsMetadata
transientSettingsMetadata,
hashesOfConsistentSettings,
clusterStateCustomMetadataMap
);
}

Expand Down
Loading

0 comments on commit 00f0249

Please sign in to comment.