Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(restore-test): Use separate map directory for each group in offline restore #8047

Merged
merged 1 commit into from
Sep 22, 2021
Merged

fix(restore-test): Use separate map directory for each group in offline restore #8047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dgraph/cmd/debug/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ type flagOptions struct {
namespace uint64
key x.Sensitive
onlySummary bool
magic uint16

// Options related to the WAL.
wdir string
Expand Down Expand Up @@ -108,6 +109,7 @@ func init() {
"Show a histogram of the key and value sizes.")
flag.BoolVar(&opt.onlySummary, "only-summary", false,
"If true, only show the summary of the p directory.")
flag.Uint16Var(&opt.magic, "magic", 1, "Magic version of the p directory.")

// Flags related to WAL.
flag.StringVarP(&opt.wdir, "wal", "w", "", "Directory where Raft write-ahead logs are stored.")
Expand Down Expand Up @@ -878,6 +880,7 @@ func run() {
WithEncryptionKey(opt.key).
WithBlockCacheSize(1 << 30).
WithIndexCacheSize(1 << 30).
WithExternalMagic(opt.magic).
WithNamespaceOffset(x.NamespaceOffset) // We don't want to see the banned data.

x.AssertTruef(len(bopts.Dir) > 0, "No posting or wal dir specified.")
Expand Down
10 changes: 6 additions & 4 deletions worker/online_restore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -542,10 +542,6 @@ func RunOfflineRestore(dir, location, backupId string, keyFile string,
}
}

mapDir, err := ioutil.TempDir(x.WorkerConfig.TmpDir, "restore-map")
x.Check(err)
defer os.RemoveAll(mapDir)

for gid := range manifest.Groups {
req := &pb.RestoreRequest{
Location: location,
Expand All @@ -554,6 +550,12 @@ func RunOfflineRestore(dir, location, backupId string, keyFile string,
EncryptionKeyFile: keyFile,
RestoreTs: 1,
}
mapDir, err := ioutil.TempDir(x.WorkerConfig.TmpDir, "restore-map")
if err != nil {
return LoadResult{Err: errors.Wrapf(err, "Failed to create temp map directory")}
}
defer os.RemoveAll(mapDir)

if _, err := RunMapper(req, mapDir); err != nil {
return LoadResult{Err: errors.Wrap(err, "RunRestore failed to map")}
}
Expand Down
8 changes: 4 additions & 4 deletions worker/restore_map.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,11 +305,11 @@ func (p *processor) processKV(buf *z.Buffer, in *loadBackupInput, kv *bpb.KV) er
toBuffer := func(kv *bpb.KV, version uint64) error {
key := y.KeyWithTs(kv.Key, version)
sz := kv.Size()
buf := buf.SliceAllocate(2 + len(key) + sz)
b := buf.SliceAllocate(2 + len(key) + sz)

binary.BigEndian.PutUint16(buf[0:2], uint16(len(key)))
x.AssertTrue(copy(buf[2:], key) == len(key))
_, err := kv.MarshalToSizedBuffer(buf[2+len(key):])
binary.BigEndian.PutUint16(b[0:2], uint16(len(key)))
x.AssertTrue(copy(b[2:], key) == len(key))
_, err := kv.MarshalToSizedBuffer(b[2+len(key):])
return err
}
if len(kv.GetUserMeta()) != 1 {
Expand Down