kvserver: use background context when applying snapshots

pkg/kv/kvserver/store_raft.go

@@ -336,14 +336,14 @@ func (s *Store) processRaftSnapshotRequest(
 		var expl string
 		var err error
 		stats, expl, err = r.handleRaftReadyRaftMuLocked(ctx, inSnap)
+		maybeFatalOnRaftReadyErr(ctx, expl, err)
 		if !stats.snap.applied {
 			// This line would be hit if a snapshot was sent when it isn't necessary
 			// (i.e. follower was able to catch up via the log in the interim) or when
 			// multiple snapshots raced (as is possible when raft leadership changes
 			// and both the old and new leaders send snapshots).
 			log.Infof(ctx, "ignored stale snapshot at index %d", snapHeader.RaftMessageRequest.Message.Snapshot.Metadata.Index)
 		}
-		maybeFatalOnRaftReadyErr(ctx, expl, err)
 		return nil
 	})
 }

pkg/kv/kvserver/store_snapshot.go

@@ -660,7 +660,13 @@ func (s *Store) receiveSnapshot(
 		return err
 	}
 	inSnap.placeholder = placeholder
-	if err := s.processRaftSnapshotRequest(ctx, header, inSnap); err != nil {
+
+	// Use a background context for applying the snapshot, as handleRaftReady is
+	// not prepared to deal with arbitrary context cancellation. Also, we've
+	// already received the entire snapshot here, so there's no point in
+	// abandoning application half-way through if the caller goes away.
+	applyCtx := s.AnnotateCtx(context.Background())
+	if err := s.processRaftSnapshotRequest(applyCtx, header, inSnap); err != nil {
 		return sendSnapshotError(stream, errors.Wrap(err.GoError(), "failed to apply snapshot"))
 	}
 	return stream.Send(&SnapshotResponse{Status: SnapshotResponse_APPLIED})