Skip to content

Commit

Permalink
Revert "Remove multiple paths from elasticsearch-node tool (elastic#7…
Browse files Browse the repository at this point in the history 
…2351)"

This reverts commit 619a011.

relates elastic#78525
relates elastic#71205
  • Loading branch information
@rjernst
rjernst committed Oct 4, 2021
1 parent 1aa5454 commit 62f3462
Show file tree
Hide file tree
Showing 11 changed files with 85 additions and 74 deletions.
2 changes: 1 addition & 1 deletion ...rnalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,7 @@ public void testResolvePath() throws Exception {
final Path indexPath = indexPathByNodeName.get(nodeName);
final OptionSet options = parser.parse("--dir", indexPath.toAbsolutePath().toString());
command.findAndProcessShardPath(options, environmentByNodeName.get(nodeName),
environmentByNodeName.get(nodeName).dataFile(),
new Path[] { environmentByNodeName.get(nodeName).dataFile() },
state, shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath)));
}
}
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ public DetachClusterCommand() {


@Override
protected void processNodePaths(Terminal terminal, Path dataPath, OptionSet options, Environment env) throws IOException {
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPath);
protected void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env) throws IOException {
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);

terminal.println(Terminal.Verbosity.VERBOSE, "Loading cluster state");
final ClusterState oldClusterState = loadTermAndClusterState(persistedClusterStateService, env).v2();
Expand Down
19 changes: 12 additions & 7 deletions server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.Map;

Expand Down Expand Up @@ -99,14 +100,14 @@ public ElasticsearchNodeCommand(String description) {
super(description);
}

public static PersistedClusterStateService createPersistedClusterStateService(Settings settings, Path dataPath) throws IOException {
final NodeMetadata nodeMetadata = PersistedClusterStateService.nodeMetadata(dataPath);
public static PersistedClusterStateService createPersistedClusterStateService(Settings settings, Path... dataPaths) throws IOException {
final NodeMetadata nodeMetadata = PersistedClusterStateService.nodeMetadata(dataPaths);
if (nodeMetadata == null) {
throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG);
}

String nodeId = nodeMetadata.nodeId();
return new PersistedClusterStateService(new Path[] { dataPath }, nodeId, namedXContentRegistry, BigArrays.NON_RECYCLING_INSTANCE,
return new PersistedClusterStateService(dataPaths, nodeId, namedXContentRegistry, BigArrays.NON_RECYCLING_INSTANCE,
new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L);
}

Expand All @@ -133,7 +134,7 @@ protected void processNodePaths(Terminal terminal, OptionSet options, Environmen
if (dataPath == null) {
throw new ElasticsearchException(NO_NODE_FOLDER_FOUND_MSG);
}
processNodePaths(terminal, dataPath.path, options, env);
processNodePaths(terminal, new Path[] { dataPath.path }, options, env);
} catch (LockObtainFailedException e) {
throw new ElasticsearchException(FAILED_TO_OBTAIN_NODE_LOCK_MSG, e);
}
Expand Down Expand Up @@ -169,14 +170,18 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) {
/**
* Process the paths. Locks for the paths is held during this method invocation.
* @param terminal the terminal to use for messages
* @param dataPath the path of the node to process
* @param dataPaths the paths of the node to process
* @param options the command line options
* @param env the env of the node to process
*/
protected abstract void processNodePaths(Terminal terminal, Path dataPath, OptionSet options, Environment env)
protected abstract void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env)
throws IOException, UserException;

protected static NodeEnvironment.NodePath createNodePath(Path path) {
protected NodeEnvironment.NodePath[] toNodePaths(Path[] dataPaths) {
return Arrays.stream(dataPaths).map(ElasticsearchNodeCommand::createNodePath).toArray(NodeEnvironment.NodePath[]::new);
}

private static NodeEnvironment.NodePath createNodePath(Path path) {
try {
return new NodeEnvironment.NodePath(path);
} catch (IOException e) {
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@ public RemoveCustomsCommand() {
}

@Override
protected void processNodePaths(Terminal terminal, Path dataPath, OptionSet options, Environment env)
protected void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env)
throws IOException, UserException {
final List<String> customsToRemove = arguments.values(options);
if (customsToRemove.isEmpty()) {
throw new UserException(ExitCodes.USAGE, "Must supply at least one custom metadata name to remove");
}

final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPath);
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);

terminal.println(Terminal.Verbosity.VERBOSE, "Loading cluster state");
final Tuple<Long, ClusterState> termAndClusterState = loadTermAndClusterState(persistedClusterStateService, env);
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@ public RemoveSettingsCommand() {
}

@Override
protected void processNodePaths(Terminal terminal, Path dataPath, OptionSet options, Environment env)
protected void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env)
throws IOException, UserException {
final List<String> settingsToRemove = arguments.values(options);
if (settingsToRemove.isEmpty()) {
throw new UserException(ExitCodes.USAGE, "Must supply at least one setting to remove");
}

final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPath);
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);

terminal.println(Terminal.Verbosity.VERBOSE, "Loading cluster state");
final Tuple<Long, ClusterState> termAndClusterState = loadTermAndClusterState(persistedClusterStateService, env);
Expand Down
4 changes: 2 additions & 2 deletions ...er/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) {
return true;
}

protected void processNodePaths(Terminal terminal, Path dataPath, OptionSet options, Environment env) throws IOException {
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPath);
protected void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env) throws IOException {
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);

final Tuple<Long, ClusterState> state = loadTermAndClusterState(persistedClusterStateService, env);
final ClusterState oldClusterState = state.v2();
Expand Down
29 changes: 15 additions & 14 deletions server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import static org.elasticsearch.env.NodeEnvironment.INDICES_FOLDER;
Expand Down Expand Up @@ -62,28 +63,28 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) {
}

@Override
protected void processNodePaths(Terminal terminal, Path dataPath, OptionSet options, Environment env) throws IOException {
protected void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env) throws IOException {
assert DiscoveryNode.canContainData(env.settings()) == false;

if (DiscoveryNode.isMasterNode(env.settings()) == false) {
processNoMasterNoDataNode(terminal, dataPath, env);
processNoMasterNoDataNode(terminal, dataPaths, env);
} else {
processMasterNoDataNode(terminal, dataPath, env);
processMasterNoDataNode(terminal, dataPaths, env);
}
}

private void processNoMasterNoDataNode(Terminal terminal, Path dataPath, Environment env) throws IOException {
NodeEnvironment.NodePath nodePath = createNodePath(dataPath);
private void processNoMasterNoDataNode(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
NodeEnvironment.NodePath[] nodePaths = toNodePaths(dataPaths);

terminal.println(Terminal.Verbosity.VERBOSE, "Collecting shard data paths");
List<Path> shardDataPaths = NodeEnvironment.collectShardDataPaths(nodePath);
List<Path> shardDataPaths = NodeEnvironment.collectShardDataPaths(nodePaths[0]);

terminal.println(Terminal.Verbosity.VERBOSE, "Collecting index metadata paths");
List<Path> indexMetadataPaths = NodeEnvironment.collectIndexMetadataPaths(nodePath);
List<Path> indexMetadataPaths = NodeEnvironment.collectIndexMetadataPaths(nodePaths[0]);

Set<Path> indexPaths = uniqueParentPaths(shardDataPaths, indexMetadataPaths);

final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPath);
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);

final Metadata metadata = loadClusterState(terminal, env, persistedClusterStateService).metadata();
if (indexPaths.isEmpty() && metadata.indices().isEmpty()) {
Expand All @@ -105,23 +106,23 @@ private void processNoMasterNoDataNode(Terminal terminal, Path dataPath, Environ

removePaths(terminal, indexPaths); // clean-up shard dirs
// clean-up all metadata dirs
MetadataStateFormat.deleteMetaState(dataPath);
IOUtils.rm(dataPath.resolve(INDICES_FOLDER));
MetadataStateFormat.deleteMetaState(dataPaths);
IOUtils.rm(Stream.of(dataPaths).map(path -> path.resolve(INDICES_FOLDER)).toArray(Path[]::new));

terminal.println("Node successfully repurposed to no-master and no-data.");
}

private void processMasterNoDataNode(Terminal terminal, Path dataPath, Environment env) throws IOException {
NodeEnvironment.NodePath nodePath = createNodePath(dataPath);
private void processMasterNoDataNode(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
NodeEnvironment.NodePath[] nodePaths = toNodePaths(dataPaths);

terminal.println(Terminal.Verbosity.VERBOSE, "Collecting shard data paths");
List<Path> shardDataPaths = NodeEnvironment.collectShardDataPaths(nodePath);
List<Path> shardDataPaths = NodeEnvironment.collectShardDataPaths(nodePaths[0]);
if (shardDataPaths.isEmpty()) {
terminal.println(NO_SHARD_DATA_TO_CLEAN_UP_FOUND);
return;
}

final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPath);
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);

final Metadata metadata = loadClusterState(terminal, env, persistedClusterStateService).metadata();

Expand Down
9 changes: 5 additions & 4 deletions server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;

public class OverrideNodeVersionCommand extends ElasticsearchNodeCommand {
private static final String TOO_NEW_MESSAGE =
Expand Down Expand Up @@ -57,9 +58,9 @@ public OverrideNodeVersionCommand() {
}

@Override
protected void processNodePaths(Terminal terminal, Path dataPath, OptionSet options, Environment env) throws IOException {
final Path nodePath = createNodePath(dataPath).path;
final NodeMetadata nodeMetadata = PersistedClusterStateService.nodeMetadata(nodePath);
protected void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env) throws IOException {
final Path[] nodePaths = Arrays.stream(toNodePaths(dataPaths)).map(p -> p.path).toArray(Path[]::new);
final NodeMetadata nodeMetadata = PersistedClusterStateService.nodeMetadata(nodePaths);
if (nodeMetadata == null) {
throw new ElasticsearchException(NO_METADATA_MESSAGE);
}
Expand All @@ -77,7 +78,7 @@ protected void processNodePaths(Terminal terminal, Path dataPath, OptionSet opti
.replace("V_NEW", nodeMetadata.nodeVersion().toString())
.replace("V_CUR", Version.CURRENT.toString()));

PersistedClusterStateService.overrideVersion(Version.CURRENT, dataPath);
PersistedClusterStateService.overrideVersion(Version.CURRENT, dataPaths);

terminal.println(SUCCESS_MESSAGE);
}
Expand Down
29 changes: 16 additions & 13 deletions server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ protected Path getPath(String dirValue) {
return PathUtils.get(dirValue, "", "");
}

protected void findAndProcessShardPath(OptionSet options, Environment environment, Path dataPath, ClusterState clusterState,
protected void findAndProcessShardPath(OptionSet options, Environment environment, Path[] dataPaths, ClusterState clusterState,
CheckedConsumer<ShardPath, IOException> consumer)
throws IOException {
final Settings settings = environment.settings();
Expand Down Expand Up @@ -154,15 +154,18 @@ protected void findAndProcessShardPath(OptionSet options, Environment environmen
final Index index = indexMetadata.getIndex();
final ShardId shId = new ShardId(index, shardId);

final Path shardPathLocation = dataPath
.resolve(NodeEnvironment.INDICES_FOLDER)
.resolve(index.getUUID())
.resolve(Integer.toString(shId.id()));
if (Files.exists(shardPathLocation)) {
final ShardPath shardPath = ShardPath.loadShardPath(logger, shId, indexSettings.customDataPath(),
new Path[]{shardPathLocation}, dataPath);
if (shardPath != null) {
consumer.accept(shardPath);
for (Path dataPath : dataPaths) {
final Path shardPathLocation = dataPath
.resolve(NodeEnvironment.INDICES_FOLDER)
.resolve(index.getUUID())
.resolve(Integer.toString(shId.id()));
if (Files.exists(shardPathLocation)) {
final ShardPath shardPath = ShardPath.loadShardPath(logger, shId, indexSettings.customDataPath(),
new Path[]{shardPathLocation}, dataPath);
if (shardPath != null) {
consumer.accept(shardPath);
return;
}
}
}
}
Expand Down Expand Up @@ -225,13 +228,13 @@ private void warnAboutIndexBackup(Terminal terminal) {

// Visible for testing
@Override
public void processNodePaths(Terminal terminal, Path dataPath, OptionSet options, Environment environment) throws IOException {
public void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment environment) throws IOException {
warnAboutIndexBackup(terminal);

final ClusterState clusterState =
loadTermAndClusterState(createPersistedClusterStateService(environment.settings(), dataPath), environment).v2();
loadTermAndClusterState(createPersistedClusterStateService(environment.settings(), dataPaths), environment).v2();

findAndProcessShardPath(options, environment, dataPath, clusterState, shardPath -> {
findAndProcessShardPath(options, environment, dataPaths, clusterState, shardPath -> {
final Path indexPath = shardPath.resolveIndex();
final Path translogPath = shardPath.resolveTranslog();
if (Files.exists(translogPath) == false || Files.isDirectory(translogPath) == false) {
Expand Down
Loading

0 comments on commit 62f3462

Please sign in to comment.