diff --git a/src/Raven.Client/ServerWide/Commands/WaitForRaftIndexCommand.cs b/src/Raven.Client/ServerWide/Commands/WaitForRaftIndexCommand.cs index 02711b613b1b..8c0be7fc19be 100644 --- a/src/Raven.Client/ServerWide/Commands/WaitForRaftIndexCommand.cs +++ b/src/Raven.Client/ServerWide/Commands/WaitForRaftIndexCommand.cs @@ -13,6 +13,11 @@ public WaitForRaftIndexCommand(long index) _index = index; } + public WaitForRaftIndexCommand(long index, string node) : this(index) + { + SelectedNodeTag = node; + } + public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url) { url = $"{node.Url}/rachis/waitfor?index={_index}"; diff --git a/src/Raven.Server/Extensions/HttpExtensions.cs b/src/Raven.Server/Extensions/HttpExtensions.cs index 449b49ef32f1..eb36351ad984 100644 --- a/src/Raven.Server/Extensions/HttpExtensions.cs +++ b/src/Raven.Server/Extensions/HttpExtensions.cs @@ -24,7 +24,7 @@ public static string GetFullUrl(this HttpRequest request) throw new InvalidOperationException("Missing Host"); string path = (request.PathBase.HasValue || request.Path.HasValue) ? (request.PathBase + request.Path).ToString() : "/"; - return request.Scheme + "://" + request.Host + path + request.Query; + return request.Scheme + "://" + request.Host + path + request.QueryString; } public static string ExtractNodeUrlFromRequest(HttpRequest request) diff --git a/src/Raven.Server/Web/RequestHandler.cs b/src/Raven.Server/Web/RequestHandler.cs index af0417eddfe9..9258a852e58d 100644 --- a/src/Raven.Server/Web/RequestHandler.cs +++ b/src/Raven.Server/Web/RequestHandler.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.IO; using System.IO.Compression; +using System.Linq; using System.Net; using System.Net.Http; using System.Runtime.CompilerServices; @@ -198,66 +199,55 @@ protected async Task WaitForExecutionOnRelevantNodes(JsonOperationContext contex if (members.Count == 0) throw new InvalidOperationException("Cannot wait for execution when there are no nodes to execute ON."); - var executors = new List(); - - try + using (var cts = CreateHttpRequestBoundTimeLimitedOperationToken(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan)) + using (var requestExecutor = ClusterRequestExecutor.Create(clusterTopology.Members.Values.ToArray(), ServerStore.Server.Certificate.Certificate, + DocumentConventions.DefaultForServer)) { - using (var cts = CancellationTokenSource.CreateLinkedTokenSource(ServerStore.ServerShutdown)) - { - cts.CancelAfter(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan); - - var waitingTasks = new List>(); - List exceptions = null; + var waitingTasks = new List>(); + List exceptions = null; - foreach (var member in members) - { - var url = clusterTopology.GetUrlFromTag(member); - var executor = ClusterRequestExecutor.CreateForSingleNode(url, ServerStore.Server.Certificate.Certificate, DocumentConventions.DefaultForServer); - executors.Add(executor); - waitingTasks.Add(ExecuteTask(executor, member, cts.Token)); - } + foreach (var member in members) + { + var url = clusterTopology.GetUrlFromTag(member); + waitingTasks.Add(ExecuteTask(requestExecutor, member, cts.Token)); + } - while (waitingTasks.Count > 0) - { - var task = await Task.WhenAny(waitingTasks); - waitingTasks.Remove(task); + while (waitingTasks.Count > 0) + { + var task = await Task.WhenAny(waitingTasks); + waitingTasks.Remove(task); - if (task.Result == null) - continue; + if (task.Result == null) + continue; - var exception = task.Result.ExtractSingleInnerException(); + var exception = task.Result.ExtractSingleInnerException(); - if (exceptions == null) - exceptions = new List(); + if (exceptions == null) + exceptions = new List(); - exceptions.Add(exception); - } + exceptions.Add(exception); + } - if (exceptions != null) + if (exceptions != null) + { + var allTimeouts = true; + foreach (var exception in exceptions) { - var allTimeouts = true; - foreach (var exception in exceptions) - { - if (exception is OperationCanceledException) - continue; + if (exception is OperationCanceledException) + continue; - allTimeouts = false; - } + allTimeouts = false; + } - var aggregateException = new AggregateException(exceptions); + var aggregateException = new AggregateException(exceptions); - if (allTimeouts) - throw new TimeoutException($"Waited too long for the raft command (number {index}) to be executed on any of the relevant nodes to this command.", aggregateException); + if (allTimeouts) + throw new TimeoutException($"Waited too long for the raft command (number {index}) to be executed on any of the relevant nodes to this command.", + aggregateException); - throw new InvalidDataException($"The database '{database}' was created but is not accessible, because all of the nodes on which this database was supposed to reside on, threw an exception.", aggregateException); - } - } - } - finally - { - foreach (var executor in executors) - { - executor.Dispose(); + throw new InvalidDataException( + $"The database '{database}' was created but is not accessible, because all of the nodes on which this database was supposed to reside on, threw an exception.", + aggregateException); } } @@ -265,7 +255,8 @@ async Task ExecuteTask(RequestExecutor executor, string nodeTag, Canc { try { - await executor.ExecuteAsync(new WaitForRaftIndexCommand(index), context, token: token); + var cmd = new WaitForRaftIndexCommand(index, nodeTag); + await executor.ExecuteAsync(cmd, context, token: token); return null; } catch (RavenException re) when (re.InnerException is HttpRequestException) diff --git a/src/Raven.Server/Web/System/AdminDatabasesHandler.cs b/src/Raven.Server/Web/System/AdminDatabasesHandler.cs index dbb661982028..a5fc995e66eb 100644 --- a/src/Raven.Server/Web/System/AdminDatabasesHandler.cs +++ b/src/Raven.Server/Web/System/AdminDatabasesHandler.cs @@ -681,7 +681,7 @@ public async Task Delete() { await ServerStore.EnsureNotPassiveAsync(); - var waitOnRecordDeletion = new List(); + var waitOnDeletion = new List(); var pendingDeletes = new HashSet(StringComparer.OrdinalIgnoreCase); var databasesToDelete = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -752,14 +752,9 @@ public async Task Delete() pendingDeletes.Add(node); topology.RemoveFromTopology(node); } - - if (topology.Count == 0) - waitOnRecordDeletion.Add(databaseName); - - continue; } - waitOnRecordDeletion.Add(databaseName); + waitOnDeletion.Add(databaseName); } } @@ -779,23 +774,46 @@ public async Task Delete() } await ServerStore.Cluster.WaitForIndexNotification(index); - long actualDeletionIndex = index; var timeToWaitForConfirmation = parameters.TimeToWaitForConfirmation ?? TimeSpan.FromSeconds(15); var sp = Stopwatch.StartNew(); int databaseIndex = 0; - while (waitOnRecordDeletion.Count > databaseIndex) + + + while (waitOnDeletion.Count > databaseIndex) { - var databaseName = waitOnRecordDeletion[databaseIndex]; + var databaseName = waitOnDeletion[databaseIndex]; using (context.OpenReadTransaction()) + using (var raw = ServerStore.Cluster.ReadRawDatabaseRecord(context, databaseName)) { - if (ServerStore.Cluster.DatabaseExists(context, databaseName) == false) + if (raw == null) { - waitOnRecordDeletion.RemoveAt(databaseIndex); + waitOnDeletion.RemoveAt(databaseIndex); continue; } + + if (parameters.FromNodes != null && parameters.FromNodes.Length > 0) + { + { + var allNodesDeleted = true; + foreach (var node in parameters.FromNodes) + { + if (raw.DeletionInProgress.ContainsKey(node) == false) + continue; + + allNodesDeleted = false; + break; + } + + if (allNodesDeleted) + { + waitOnDeletion.RemoveAt(databaseIndex); + continue; + } + } + } } // we'll now wait for the _next_ operation in the cluster // since deletion involve multiple operations in the cluster @@ -820,6 +838,11 @@ public async Task Delete() } } + if (parameters.FromNodes != null && parameters.FromNodes.Length > 0) + { + await WaitForExecutionOnRelevantNodes(context, "server", ServerStore.GetClusterTopology(), parameters.FromNodes.ToList(), actualDeletionIndex); + } + await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue