Skip to content
This repository has been archived by the owner on Jul 24, 2024. It is now read-only.

backup: optimize backupmeta to reduce memory useage. #1171

Merged
merged 45 commits into from
Jun 10, 2021
Merged

backup: optimize backupmeta to reduce memory useage. #1171

Show file tree
Hide file tree
Changes from 40 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
61ac4fa
--wip-- [skip ci]
overvenus Mar 16, 2021
cb58419
--wip-- [skip ci]
overvenus Mar 18, 2021
0af574e
--wip-- [skip ci]
overvenus Mar 24, 2021
16ebf5c
update some logic for datafile and schemas
3pointer May 26, 2021
23ca7ea
debug
3pointer May 27, 2021
42ea4b6
tmp add
3pointer May 31, 2021
08c004d
update metaWriter for compatibility
3pointer May 31, 2021
ac3abac
remove useless code
3pointer May 31, 2021
f770b0c
fix sync
3pointer May 31, 2021
8111ae0
fix lock
3pointer May 31, 2021
1d7ae5a
fix
3pointer May 31, 2021
6eaa9e9
fix
3pointer May 31, 2021
db96bcd
Merge branch 'master' into backupmetav2
3pointer May 31, 2021
fb04906
update restore load from backupmeta
3pointer May 31, 2021
3bcc78b
fix
3pointer Jun 1, 2021
1544109
fix lost files
3pointer Jun 1, 2021
4e88858
fix dead lock
3pointer Jun 1, 2021
b1319ec
fix no result
3pointer Jun 1, 2021
7007fb7
fix read ddl
3pointer Jun 1, 2021
0af8d6d
make ci happy
3pointer Jun 1, 2021
8db563c
update kvproto
3pointer Jun 2, 2021
033caf9
make ci happy
3pointer Jun 2, 2021
504b1a7
make ci happy
3pointer Jun 2, 2021
98dd440
update
3pointer Jun 2, 2021
46d1fdc
fix lost
3pointer Jun 2, 2021
83d0a3e
fix ci
3pointer Jun 2, 2021
82ceb71
fix partition
3pointer Jun 3, 2021
6ac6a39
fix lost
3pointer Jun 3, 2021
ba20a31
fix log
3pointer Jun 3, 2021
03edcd8
fix build
3pointer Jun 3, 2021
adc9bc6
fix systable schema
3pointer Jun 3, 2021
373809d
fix ddls
3pointer Jun 3, 2021
7318c79
fix build
3pointer Jun 3, 2021
bd73a59
make format
3pointer Jun 3, 2021
1e29ba2
fix build
3pointer Jun 3, 2021
6a58d0e
Merge branch 'master' into backupmetav2
3pointer Jun 3, 2021
f4e439e
fix br_s3 test
3pointer Jun 3, 2021
138a92f
Update pkg/metautil/metafile.go
3pointer Jun 4, 2021
4640485
address comemnt
3pointer Jun 4, 2021
e7c166a
address comments
3pointer Jun 5, 2021
707b45b
Merge branch 'master' into backupmetav2
3pointer Jun 9, 2021
528f192
update kvproto && address comment
3pointer Jun 9, 2021
3e3b0c6
fix build
3pointer Jun 9, 2021
b9c4ea6
fix build
3pointer Jun 9, 2021
a3ab71d
fix test
3pointer Jun 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions cmd/br/debug.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"path"
"reflect"

"github.com/pingcap/br/pkg/metautil"
3pointer marked this conversation as resolved.
Show resolved Hide resolved

"github.com/gogo/protobuf/proto"
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/backup"
Expand Down Expand Up @@ -72,12 +74,13 @@ func newCheckSumCommand() *cobra.Command {
return errors.Trace(err)
}

_, s, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
_, s, backupMeta, err := task.ReadBackupMeta(ctx, metautil.MetaFile, &cfg)
if err != nil {
return errors.Trace(err)
}

dbs, err := utils.LoadBackupTables(backupMeta)
reader := metautil.NewMetaReader(backupMeta, s)
dbs, err := utils.LoadBackupTables(ctx, reader)
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -170,18 +173,19 @@ func newBackupMetaValidateCommand() *cobra.Command {
if err = cfg.ParseFromFlags(cmd.Flags()); err != nil {
return errors.Trace(err)
}
_, _, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
_, s, backupMeta, err := task.ReadBackupMeta(ctx, metautil.MetaFile, &cfg)
if err != nil {
log.Error("read backupmeta failed", zap.Error(err))
return errors.Trace(err)
}
dbs, err := utils.LoadBackupTables(backupMeta)
reader := metautil.NewMetaReader(backupMeta, s)
dbs, err := utils.LoadBackupTables(ctx, reader)
if err != nil {
log.Error("load tables failed", zap.Error(err))
return errors.Trace(err)
}
files := make([]*backuppb.File, 0)
tables := make([]*utils.Table, 0)
tables := make([]*metautil.Table, 0)
for _, db := range dbs {
for _, table := range db.Tables {
files = append(files, table.Files...)
Expand Down Expand Up @@ -261,7 +265,7 @@ func decodeBackupMetaCommand() *cobra.Command {
if err := cfg.ParseFromFlags(cmd.Flags()); err != nil {
return errors.Trace(err)
}
_, s, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
_, s, backupMeta, err := task.ReadBackupMeta(ctx, metautil.MetaFile, &cfg)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -273,11 +277,11 @@ func decodeBackupMetaCommand() *cobra.Command {
if err != nil {
return errors.Trace(err)
}
err = s.WriteFile(ctx, utils.MetaJSONFile, backupMetaJSON)
err = s.WriteFile(ctx, metautil.MetaJSONFile, backupMetaJSON)
if err != nil {
return errors.Trace(err)
}
cmd.Printf("backupmeta decoded at %s\n", path.Join(cfg.Storage, utils.MetaJSONFile))
cmd.Printf("backupmeta decoded at %s\n", path.Join(cfg.Storage, metautil.MetaJSONFile))
return nil
}

Expand Down Expand Up @@ -327,7 +331,7 @@ func encodeBackupMetaCommand() *cobra.Command {
return errors.Trace(err)
}

metaData, err := s.ReadFile(ctx, utils.MetaJSONFile)
metaData, err := s.ReadFile(ctx, metautil.MetaJSONFile)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -341,7 +345,7 @@ func encodeBackupMetaCommand() *cobra.Command {
return errors.Trace(err)
}

fileName := utils.MetaFile
fileName := metautil.MetaFile
if ok, _ := s.FileExists(ctx, fileName); ok {
// Do not overwrite origin meta file
fileName += "_from_json"
Expand Down
5 changes: 5 additions & 0 deletions errors.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ error = '''
invalid argument
'''

["BR:Common:ErrInvalidMetaFile"]
error = '''
invalid metafile
'''

["BR:Common:ErrUndefinedDbOrTable"]
error = '''
undefined restore databases or tables
Expand Down
5 changes: 4 additions & 1 deletion go.mod1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@ require (
github.com/go-sql-driver/mysql v1.5.0
github.com/gogo/protobuf v1.3.2
github.com/golang/mock v1.4.4
github.com/golang/protobuf v1.3.4
github.com/google/btree v1.0.0
github.com/google/uuid v1.1.1
github.com/joho/sqltocsv v0.0.0-20210208114054-cb2c3a95fb99
github.com/opentracing/opentracing-go v1.1.0
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3
github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd
github.com/pingcap/kvproto v0.0.0-20210507074444-0ec2d0dc2e4b
github.com/pingcap/kvproto v0.0.0-20210507054410-a8152f8a876c
github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4
github.com/pingcap/parser v0.0.0-20210513020953-ae2c4497c07b
github.com/pingcap/tidb v1.1.0-beta.0.20210517044538-8ad868f801fc
Expand Down Expand Up @@ -55,3 +56,5 @@ require (
)

replace cloud.google.com/go/storage => github.com/3pointer/google-cloud-go/storage v1.6.1-0.20210108125931-b59bfa0720b2

replace github.com/pingcap/kvproto => github.com/3pointer/kvproto v0.0.0-20210602032321-0dbb9be4f2af
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the corresponding kvproto PR?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

corredponding PR : https://github.com/3pointer/kvproto/tree/backupmetav2
the master kvproto has this breaking change. pingcap/kvproto#765, But the latest tidb doesn't. so I need to revert it until pingcap/tidb#25102 merged.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😭

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall we push tidb to resolve or rollback the corresponding issue?

12 changes: 2 additions & 10 deletions go.sum1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/3pointer/google-cloud-go/storage v1.6.1-0.20210108125931-b59bfa0720b2 h1:Bnm+q0FAI1AXoPX3cPIi2/a4sg2J2f/D0yPr+7EMPg0=
github.com/3pointer/google-cloud-go/storage v1.6.1-0.20210108125931-b59bfa0720b2/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
github.com/3pointer/kvproto v0.0.0-20210602032321-0dbb9be4f2af h1:AtAMSv9qWhxJjnnnemluwH2AGf3ZMvWo5ZktU0vX17s=
github.com/3pointer/kvproto v0.0.0-20210602032321-0dbb9be4f2af/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
Expand Down Expand Up @@ -212,7 +214,6 @@ github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwn
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk=
github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
Expand All @@ -235,7 +236,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -445,12 +445,6 @@ github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 h1:Pe2LbxRmbTfAoKJ65bZL
github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20210219064844-c1844a4775d6/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20210507054410-a8152f8a876c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20210507074444-0ec2d0dc2e4b h1:e42N26QQjVA/obDrFFapJ1YLB+j5aPQOh7R+cIGR9Bk=
github.com/pingcap/kvproto v0.0.0-20210507074444-0ec2d0dc2e4b/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
Expand Down Expand Up @@ -866,7 +860,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
Expand All @@ -883,7 +876,6 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 h1:YzfoEYWbODU5Fbt37+h7X16BWQbad7Q4S6gclTKFXM8=
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand Down
Loading