Skip to content

Commit

Permalink
fix(backup): use StreamWriter to write to badger during backup/restore
Browse files Browse the repository at this point in the history 
This commit is a major rewrite of backup and online restore
code. It used to use KVLoader in badger. Now it instead uses
StreamWriter that is much faster for writes.

cherry-pick PR #7753

following commits are cherry-picked (in reverse order):
 * opt(restore): Sort the buffer before spinning the writeToDisk goroutine (#7984) (#7996)
 * fix(backup): Fix full backup request (#7932) (#7933)
 * fix: fixing graphql schema update when the data is restored +
 * fix(restore): return nil if there is error (#7899)
        skipping /probe/graphql from audit (#7925)
 * Don't ban namespace in export_backup
 * reset the kv.StreamId before sending to stream writer (#7833) (#7837)
 * fix(restore): Bump uid and namespace after restore (#7790) (#7800)
 * fix(ee): GetKeys should return an error (#7713) (#7797)
 * fix(backup): Free the UidPack after use (#7786)
 * fix(export-backup): Fix double free in export backup (#7780) (#7783)
 * fix(lsbackup): Fix profiler in lsBackup (#7729)
 * Bring back "perf(Backup): Improve backup performance (#7601)"
 * Opt(Backup): Make backups faster (#7680)
 * Fix s3 backup copy (#7669)
 * [BREAKING] Opt(Restore): Optimize Restore's new map-reduce based design (#7666)
 * Perf(restore): Implement map-reduce based restore (#7664)
 * feat(backup): Merge backup refactoring
 * Revert "perf(Backup): Improve backup performance (#7601)"
  • Loading branch information
@ahsanbarkati @all-seeing-code
ahsanbarkati authored and all-seeing-code committed Jan 23, 2023
1 parent d3bf7b7 commit 9b20a70
Show file tree
Hide file tree
Showing 27 changed files with 92 additions and 91 deletions.
4 changes: 3 additions & 1 deletion codec/codec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,8 @@ func Decode(pack *pb.UidPack, seek uint64) []uint64 {

// DecodeToBuffer is the same as Decode but it returns a z.Buffer which is
// calloc'ed and can be SHOULD be freed up by calling buffer.Release().
func DecodeToBuffer(buf *z.Buffer, pack *pb.UidPack) {
func DecodeToBuffer(buf *z.Buffer, pack *pb.UidPack) *z.Buffer {

var last uint64
tmp := make([]byte, 16)
dec := Decoder{Pack: pack}
Expand All @@ -417,6 +418,7 @@ func DecodeToBuffer(buf *z.Buffer, pack *pb.UidPack) {
last = u
}
}
return buf
}

func match32MSB(num1, num2 uint64) bool {
Expand Down
1 change: 0 additions & 1 deletion codec/codec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ func TestBufferUidPack(t *testing.T) {
// Some edge case tests.
pack := Encode([]uint64{}, 128)
FreePack(pack)

buf := z.NewBuffer(10<<10, "TestBufferUidPack")
defer buf.Release()
DecodeToBuffer(buf, &pb.UidPack{})
Expand Down
6 changes: 3 additions & 3 deletions dgraph/cmd/zero/zero_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ import (
"math"
"testing"

"github.com/stretchr/testify/require"
"google.golang.org/grpc"

"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/testutil"

"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)

func TestRemoveNode(t *testing.T) {
Expand Down
6 changes: 3 additions & 3 deletions ee/acl/acl_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ import (
"testing"
"time"

"github.com/golang/glog"
"github.com/stretchr/testify/require"

"github.com/dgraph-io/dgo/v210"
"github.com/dgraph-io/dgo/v210/protos/api"
"github.com/dgraph-io/dgraph/testutil"
"github.com/dgraph-io/dgraph/x"

"github.com/golang/glog"
"github.com/stretchr/testify/require"
)

var (
Expand Down
16 changes: 7 additions & 9 deletions ee/backup/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,19 @@ import (
"path/filepath"
"time"

"github.com/golang/glog"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"

bpb "github.com/dgraph-io/badger/v3/pb"
"github.com/dgraph-io/dgraph/ee"
"github.com/dgraph-io/dgraph/posting"
"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/worker"
"github.com/dgraph-io/dgraph/x"
"github.com/dgraph-io/ristretto/z"

"github.com/golang/glog"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
)

// Restore is the sub-command used to restore a backup.
Expand Down Expand Up @@ -467,10 +467,8 @@ func runExportBackup() error {
if err := worker.RunReducer(w, mapDir); err != nil {
return errors.Wrap(err, "Failed to reduce the map")
}
if files, err := exportStorage.FinishWriting(writers); err != nil {
if _, err := exportStorage.FinishWriting(writers); err != nil {
return errors.Wrap(err, "Failed to finish write")
} else {
glog.Infof("done exporting files: %v\n", files)
}
}
return nil
Expand Down
4 changes: 2 additions & 2 deletions ee/vault/vault.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
package vault

import (
"github.com/dgraph-io/dgraph/ee"

"github.com/golang/glog"
"github.com/spf13/viper"

"github.com/dgraph-io/dgraph/ee"
)

func GetKeys(config *viper.Viper) (*ee.Keys, error) {
Expand Down
6 changes: 3 additions & 3 deletions ee/vault_ee.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ import (
"io/ioutil"
"reflect"

"github.com/dgraph-io/dgraph/x"
"github.com/dgraph-io/ristretto/z"

"github.com/golang/glog"
"github.com/hashicorp/vault/api"
"github.com/spf13/viper"

"github.com/dgraph-io/dgraph/x"
"github.com/dgraph-io/ristretto/z"
)

func vaultGetKeys(config *viper.Viper) (aclKey, encKey x.Sensitive) {
Expand Down
1 change: 1 addition & 0 deletions graphql/admin/schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package admin
import (
"context"
"encoding/json"
"github.com/dgraph-io/dgraph/worker"

"github.com/golang/glog"

Expand Down
2 changes: 2 additions & 0 deletions systest/backup/multi-tenancy/backup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"strings"
"testing"

"github.com/dgraph-io/dgo/v210"
"github.com/dgraph-io/dgo/v210/protos/api"
"github.com/stretchr/testify/require"

"github.com/dgraph-io/dgo/v210"
Expand Down
4 changes: 2 additions & 2 deletions worker/acl_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ package worker
import (
"sync"

"github.com/pkg/errors"

"github.com/dgraph-io/dgraph/ee/acl"
"github.com/dgraph-io/dgraph/x"

"github.com/pkg/errors"
)

// aclCache is the cache mapping group names to the corresponding group acls
Expand Down
4 changes: 2 additions & 2 deletions worker/acl_cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ package worker
import (
"testing"

"github.com/stretchr/testify/require"

"github.com/dgraph-io/dgraph/ee/acl"
"github.com/dgraph-io/dgraph/x"

"github.com/stretchr/testify/require"
)

func TestAclCache(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions worker/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ import (
"context"
"math"

"github.com/pkg/errors"

"github.com/dgraph-io/badger/v3"
"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/x"

"github.com/pkg/errors"
)

// predicateSet is a map whose keys are predicates. It is meant to be used as a set.
Expand Down
12 changes: 6 additions & 6 deletions worker/backup_ee.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,6 @@ import (
"sync"
"time"

"github.com/golang/glog"
"github.com/golang/protobuf/proto"
"github.com/golang/snappy"
"github.com/pkg/errors"
ostats "go.opencensus.io/stats"

"github.com/dgraph-io/badger/v3"
bpb "github.com/dgraph-io/badger/v3/pb"
"github.com/dgraph-io/badger/v3/y"
Expand All @@ -39,6 +33,12 @@ import (
"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/x"
"github.com/dgraph-io/ristretto/z"

"github.com/golang/glog"
"github.com/golang/protobuf/proto"
"github.com/golang/snappy"
"github.com/pkg/errors"
ostats "go.opencensus.io/stats"
)

// Backup handles a request coming from another node.
Expand Down
6 changes: 3 additions & 3 deletions worker/backup_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@ import (
"path/filepath"
"time"

"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/x"

"github.com/golang/glog"
"github.com/minio/minio-go/v6"
"github.com/minio/minio-go/v6/pkg/credentials"
"github.com/pkg/errors"

"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/x"
)

const (
Expand Down
4 changes: 2 additions & 2 deletions worker/backup_manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ import (
"sort"
"strings"

"github.com/pkg/errors"

"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/x"

"github.com/pkg/errors"
)

func verifyManifests(manifests []*Manifest) error {
Expand Down
4 changes: 2 additions & 2 deletions worker/backup_oss.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ package worker
import (
"context"

"github.com/golang/glog"

"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/x"

"github.com/golang/glog"
)

// Backup implements the Worker interface.
Expand Down
8 changes: 4 additions & 4 deletions worker/cdc_ee.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@ import (
"sync/atomic"
"time"

"github.com/golang/glog"
"github.com/pkg/errors"
"go.etcd.io/etcd/raft/raftpb"

"github.com/dgraph-io/dgraph/posting"
"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/types"
"github.com/dgraph-io/dgraph/x"
"github.com/dgraph-io/ristretto/z"

"github.com/golang/glog"
"github.com/pkg/errors"
"go.etcd.io/etcd/raft/raftpb"
)

const (
Expand Down
20 changes: 10 additions & 10 deletions worker/draft.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,6 @@ import (
"sync/atomic"
"time"

"github.com/dustin/go-humanize"
"github.com/golang/glog"
"github.com/pkg/errors"
"go.etcd.io/etcd/raft"
"go.etcd.io/etcd/raft/raftpb"
ostats "go.opencensus.io/stats"
"go.opencensus.io/tag"
otrace "go.opencensus.io/trace"
"golang.org/x/net/trace"

"github.com/dgraph-io/badger/v3"
bpb "github.com/dgraph-io/badger/v3/pb"
"github.com/dgraph-io/dgraph/conn"
Expand All @@ -48,6 +38,16 @@ import (
"github.com/dgraph-io/dgraph/types"
"github.com/dgraph-io/dgraph/x"
"github.com/dgraph-io/ristretto/z"

"github.com/dustin/go-humanize"
"github.com/golang/glog"
"github.com/pkg/errors"
"go.etcd.io/etcd/raft"
"go.etcd.io/etcd/raft/raftpb"
ostats "go.opencensus.io/stats"
"go.opencensus.io/tag"
otrace "go.opencensus.io/trace"
"golang.org/x/net/trace"
)

type operation struct {
Expand Down
6 changes: 3 additions & 3 deletions worker/draft_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ import (
"os"
"testing"

"github.com/stretchr/testify/require"
"go.etcd.io/etcd/raft/raftpb"

"github.com/dgraph-io/dgraph/posting"
"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/raftwal"
"github.com/dgraph-io/dgraph/x"

"github.com/stretchr/testify/require"
"go.etcd.io/etcd/raft/raftpb"
)

func getEntryForMutation(index, startTs uint64) raftpb.Entry {
Expand Down
8 changes: 4 additions & 4 deletions worker/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,6 @@ import (
"strings"
"time"

"github.com/golang/glog"
"github.com/minio/minio-go/v6"
"github.com/pkg/errors"

"github.com/dgraph-io/badger/v3"
bpb "github.com/dgraph-io/badger/v3/pb"
"github.com/dgraph-io/dgo/v210/protos/api"
Expand All @@ -46,6 +42,10 @@ import (
"github.com/dgraph-io/dgraph/types/facets"
"github.com/dgraph-io/dgraph/x"
"github.com/dgraph-io/ristretto/z"

"github.com/golang/glog"
"github.com/minio/minio-go/v6"
"github.com/pkg/errors"
)

// DefaultExportFormat stores the name of the default format for exports.
Expand Down
6 changes: 3 additions & 3 deletions worker/export_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,6 @@ import (
"testing"
"time"

"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/require"

"github.com/dgraph-io/dgo/v210/protos/api"
"github.com/dgraph-io/dgraph/chunker"
"github.com/dgraph-io/dgraph/dql"
Expand All @@ -46,6 +43,9 @@ import (
"github.com/dgraph-io/dgraph/types"
"github.com/dgraph-io/dgraph/types/facets"
"github.com/dgraph-io/dgraph/x"

"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/require"
)

const (
Expand Down
8 changes: 4 additions & 4 deletions worker/graphql_schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@ import (
"sync"
"time"

"github.com/golang/glog"
"github.com/pkg/errors"
"google.golang.org/grpc/metadata"

"github.com/dgraph-io/dgraph/conn"
"github.com/dgraph-io/dgraph/protos/pb"
"github.com/dgraph-io/dgraph/schema"
"github.com/dgraph-io/dgraph/x"

"github.com/golang/glog"
"github.com/pkg/errors"
"google.golang.org/grpc/metadata"
)

const (
Expand Down
Loading

0 comments on commit 9b20a70

Please sign in to comment.