Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Snapshot V2] Fix for create snapshot build and test failures #15602

Merged
merged 7 commits into from
Sep 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,27 @@
in.getRepositoryData(listener);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
long repositoryStateId,
Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
ActionListener<RepositoryData> listener
) {
in.finalizeSnapshot(

Check warning on line 110 in server/src/main/java/org/opensearch/repositories/FilterRepository.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/repositories/FilterRepository.java#L110

Added line #L110 was not covered by tests
shardGenerations,
repositoryStateId,
clusterMetadata,
snapshotInfo,
repositoryMetaVersion,
stateTransformer,
listener
);
}

Check warning on line 119 in server/src/main/java/org/opensearch/repositories/FilterRepository.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/repositories/FilterRepository.java#L119

Added line #L119 was not covered by tests

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,30 @@ default Repository create(RepositoryMetadata metadata, Function<String, Reposito
*/
void getRepositoryData(ActionListener<RepositoryData> listener);

/**
* Finalizes snapshotting process
* <p>
* This method is called on cluster-manager after all shards are snapshotted.
*
* @param shardGenerations updated shard generations
* @param repositoryStateId the unique id identifying the state of the repository when the snapshot began
* @param clusterMetadata cluster metadata
* @param snapshotInfo SnapshotInfo instance to write for this snapshot
* @param repositoryMetaVersion version of the updated repository metadata to write
* @param stateTransformer a function that filters the last cluster state update that the snapshot finalization will execute and
* is used to remove any state tracked for the in-progress snapshot from the cluster state
* @param listener listener to be invoked with the new {@link RepositoryData} after completing the snapshot
*/
void finalizeSnapshot(
ShardGenerations shardGenerations,
long repositoryStateId,
Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
ActionListener<RepositoryData> listener
);

/**
* Finalizes snapshotting process
* <p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2024,6 +2024,28 @@
return new DeleteResult(matchingShardPaths.size(), totalBytes);
}

@Override
public void finalizeSnapshot(
final ShardGenerations shardGenerations,
final long repositoryStateId,
final Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
final ActionListener<RepositoryData> listener
) {
finalizeSnapshot(

Check warning on line 2037 in server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java#L2037

Added line #L2037 was not covered by tests
shardGenerations,
repositoryStateId,
clusterMetadata,
snapshotInfo,
repositoryMetaVersion,
stateTransformer,
Priority.NORMAL,
sachinpkale marked this conversation as resolved.
Show resolved Hide resolved
listener
);
}

Check warning on line 2047 in server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java#L2047

Added line #L2047 was not covered by tests

@Override
public void finalizeSnapshot(
final ShardGenerations shardGenerations,
Expand Down
32 changes: 32 additions & 0 deletions server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,38 @@
);
}

public SnapshotInfo(
SnapshotId snapshotId,
List<String> indices,
List<String> dataStreams,
long startTime,
String reason,
long endTime,
int totalShards,
List<SnapshotShardFailure> shardFailures,
Boolean includeGlobalState,
Map<String, Object> userMetadata,
Boolean remoteStoreIndexShallowCopy
) {
this(

Check warning on line 370 in server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java#L370

Added line #L370 was not covered by tests
snapshotId,
indices,
dataStreams,
snapshotState(reason, shardFailures),

Check warning on line 374 in server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java#L374

Added line #L374 was not covered by tests
reason,
Version.CURRENT,
startTime,
endTime,
totalShards,
totalShards - shardFailures.size(),

Check warning on line 380 in server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java#L380

Added line #L380 was not covered by tests
shardFailures,
includeGlobalState,
userMetadata,
remoteStoreIndexShallowCopy,
0
);
}

Check warning on line 387 in server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java#L387

Added line #L387 was not covered by tests

public SnapshotInfo(
SnapshotId snapshotId,
List<String> indices,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
Setting.Property.Dynamic
);

private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = "__";
/**
* Setting to specify the maximum number of shards that can be included in the result for the snapshot status
* API call. Note that it does not apply to V2-shallow snapshots.
Expand All @@ -216,8 +217,6 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
Setting.Property.NodeScope,
Setting.Property.Dynamic
);

private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = ":";
private volatile int maxConcurrentOperations;

public SnapshotsService(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -681,6 +681,19 @@ public void finalizeSnapshot(
listener.onResponse(null);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
long repositoryStateId,
Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
ActionListener<RepositoryData> listener
) {
listener.onResponse(null);
}

@Override
public void deleteSnapshots(
Collection<SnapshotId> snapshotIds,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,19 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
long repositoryStateId,
Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
ActionListener<RepositoryData> listener
) {
listener.onResponse(null);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2898,7 +2898,7 @@ private static Settings buildRemoteStoreNodeAttributes(
}
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values()));
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean());
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), false);
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean());
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : "");
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : "");
return settings.build();
Expand Down
Loading