diff --git a/dm/_utils/terror_gen/errors_release.txt b/dm/_utils/terror_gen/errors_release.txt index 02941319971..06ab92f09f6 100644 --- a/dm/_utils/terror_gen/errors_release.txt +++ b/dm/_utils/terror_gen/errors_release.txt @@ -153,7 +153,7 @@ ErrConfigReadCfgFromFile,[code=20018:class=config:scope=internal:level=medium], ErrConfigNeedUniqueTaskName,[code=20019:class=config:scope=internal:level=medium], "Message: must specify a unique task name, Workaround: Please check the `name` config in task configuration file." ErrConfigInvalidTaskMode,[code=20020:class=config:scope=internal:level=medium], "Message: please specify right task-mode, support `full`, `incremental`, `all`, Workaround: Please check the `task-mode` config in task configuration file." ErrConfigNeedTargetDB,[code=20021:class=config:scope=internal:level=medium], "Message: must specify target-database, Workaround: Please check the `target-database` config in task configuration file." -ErrConfigMetadataNotSet,[code=20022:class=config:scope=internal:level=medium], "Message: mysql-instance(%d) must set meta for task-mode %s, Workaround: Please check the `meta` config in task configuration file." +ErrConfigMetadataNotSet,[code=20022:class=config:scope=internal:level=medium], "Message: mysql-instance(%s) must set meta for task-mode %s, Workaround: Please check the `meta` config in task configuration file." ErrConfigRouteRuleNotFound,[code=20023:class=config:scope=internal:level=medium], "Message: mysql-instance(%d)'s route-rules %s not exist in routes, Workaround: Please check the `route-rules` config in task configuration file." ErrConfigFilterRuleNotFound,[code=20024:class=config:scope=internal:level=medium], "Message: mysql-instance(%d)'s filter-rules %s not exist in filters, Workaround: Please check the `filter-rules` config in task configuration file." ErrConfigColumnMappingNotFound,[code=20025:class=config:scope=internal:level=medium], "Message: mysql-instance(%d)'s column-mapping-rules %s not exist in column-mapping, Workaround: Please check the `column-mapping-rules` config in task configuration file." diff --git a/dm/dm/config/task.go b/dm/dm/config/task.go index e9c4f283376..22316d0f89e 100644 --- a/dm/dm/config/task.go +++ b/dm/dm/config/task.go @@ -660,7 +660,7 @@ func (c *TaskConfig) adjust() error { } case ModeIncrement: if inst.Meta == nil { - return terror.ErrConfigMetadataNotSet.Generate(i, c.TaskMode) + return terror.ErrConfigMetadataNotSet.Generate(inst.SourceID, c.TaskMode) } err := inst.Meta.Verify() if err != nil { diff --git a/dm/dm/config/task_converters.go b/dm/dm/config/task_converters.go index 9cb6034d529..187ae9e0ef3 100644 --- a/dm/dm/config/task_converters.go +++ b/dm/dm/config/task_converters.go @@ -166,10 +166,12 @@ func OpenAPITaskToSubTaskConfigs(task *openapi.Task, toDBCfg *DBConfig, sourceCf } subTaskCfg.Meta = meta } - // check must set meta when mode is ModeIncrement + + // if there is no meta for incremental task, we print a warning log if subTaskCfg.Meta == nil && subTaskCfg.Mode == ModeIncrement { - return nil, terror.ErrConfigMetadataNotSet.Generate(i, ModeIncrement) + log.L().Warn("mysql-instance doesn't set meta for incremental mode, user should specify start_time to start task.", zap.String("sourceID", sourceCfg.SourceName)) } + // set shard config if task.ShardMode != nil { subTaskCfg.IsSharding = true @@ -691,3 +693,34 @@ func genFilterRuleName(sourceName string, idx int) string { // NOTE that we don't have user input filter rule name in sub task config, so we make one by ourself return fmt.Sprintf("%s-filter-rule-%d", sourceName, idx) } + +func OpenAPIStartTaskReqToTaskCliArgs(req openapi.StartTaskRequest) (*TaskCliArgs, error) { + if req.StartTime == nil && req.SafeModeTimeDuration == nil { + return nil, nil + } + cliArgs := &TaskCliArgs{} + if req.StartTime != nil { + cliArgs.StartTime = *req.StartTime + } + if req.SafeModeTimeDuration != nil { + cliArgs.SafeModeDuration = *req.SafeModeTimeDuration + } + + if err := cliArgs.Verify(); err != nil { + return nil, err + } + return cliArgs, nil +} + +func OpenAPIStopTaskReqToTaskCliArgs(req openapi.StopTaskRequest) (*TaskCliArgs, error) { + if req.TimeoutDuration == nil { + return nil, nil + } + cliArgs := &TaskCliArgs{ + WaitTimeOnStop: *req.TimeoutDuration, + } + if err := cliArgs.Verify(); err != nil { + return nil, err + } + return cliArgs, nil +} diff --git a/dm/dm/master/openapi_controller.go b/dm/dm/master/openapi_controller.go index 831ac1d7a36..f00d21af8f3 100644 --- a/dm/dm/master/openapi_controller.go +++ b/dm/dm/master/openapi_controller.go @@ -24,9 +24,14 @@ import ( "fmt" "strings" + clientv3 "go.etcd.io/etcd/client/v3" + "github.com/pingcap/log" "go.uber.org/zap" + "github.com/pingcap/tiflow/dm/dm/master/scheduler" + "github.com/pingcap/tiflow/dm/pkg/ha" + "github.com/pingcap/tiflow/dm/checker" dmcommon "github.com/pingcap/tiflow/dm/dm/common" "github.com/pingcap/tiflow/dm/dm/config" @@ -377,7 +382,7 @@ func (s *Server) checkOpenAPITaskBeforeOperate(ctx context.Context, task *openap if sourceCfg := s.scheduler.GetSourceCfgByID(cfg.SourceName); sourceCfg != nil { sourceCfgMap[cfg.SourceName] = sourceCfg } else { - return nil, "", terror.ErrSchedulerSourceCfgNotExist.Generate(sourceCfg.SourceID) + return nil, "", terror.ErrSchedulerSourceCfgNotExist.Generate(cfg.SourceName) } } // generate sub task configs @@ -666,6 +671,10 @@ func (s *Server) startTask(ctx context.Context, taskName string, req openapi.Sta if !ok { return terror.ErrSchedulerSourceCfgNotExist.Generate(sourceName) } + // start task check. incremental task need to specify meta or start time + if subTaskCfg.Meta == nil && subTaskCfg.Mode == config.ModeIncrement && req.StartTime == nil { + return terror.ErrConfigMetadataNotSet.Generate(sourceName, config.ModeIncrement) + } cfg := s.scheduler.GetSourceCfgByID(sourceName) if cfg == nil { return terror.ErrSchedulerSourceCfgNotExist.Generate(sourceName) @@ -679,10 +688,14 @@ func (s *Server) startTask(ctx context.Context, taskName string, req openapi.Sta return nil } - // TODO(ehco) support other start args after https://github.com/pingcap/tiflow/pull/4601 merged + var ( + release scheduler.ReleaseFunc + err error + ) + // removeMeta if req.RemoveMeta != nil && *req.RemoveMeta { // use same latch for remove-meta and start-task - release, err := s.scheduler.AcquireSubtaskLatch(taskName) + release, err = s.scheduler.AcquireSubtaskLatch(taskName) if err != nil { return terror.ErrSchedulerLatchInUse.Generate("RemoveMeta", taskName) } @@ -693,8 +706,21 @@ func (s *Server) startTask(ctx context.Context, taskName string, req openapi.Sta if err != nil { return terror.Annotate(err, "while removing metadata") } + } + + // handle task cli args + cliArgs, err := config.OpenAPIStartTaskReqToTaskCliArgs(req) + if err != nil { + return terror.Annotate(err, "while converting task command line arguments") + } + + if err = handleCliArgs(s.etcdClient, taskName, *req.SourceNameList, cliArgs); err != nil { + return err + } + if release != nil { release() } + return s.scheduler.UpdateExpectSubTaskStage(pb.Stage_Running, taskName, *req.SourceNameList...) } @@ -705,10 +731,34 @@ func (s *Server) stopTask(ctx context.Context, taskName string, req openapi.Stop sourceNameList := openapi.SourceNameList(s.getTaskSourceNameList(taskName)) req.SourceNameList = &sourceNameList } - // TODO(ehco): support stop req after https://github.com/pingcap/tiflow/pull/4601 merged + // handle task cli args + cliArgs, err := config.OpenAPIStopTaskReqToTaskCliArgs(req) + if err != nil { + return terror.Annotate(err, "while converting task command line arguments") + } + if err = handleCliArgs(s.etcdClient, taskName, *req.SourceNameList, cliArgs); err != nil { + return err + } return s.scheduler.UpdateExpectSubTaskStage(pb.Stage_Stopped, taskName, *req.SourceNameList...) } +// handleCliArgs handles cli args. +// it will try to delete args if cli args is nil. +func handleCliArgs(cli *clientv3.Client, taskName string, sources []string, cliArgs *config.TaskCliArgs) error { + if cliArgs == nil { + err := ha.DeleteTaskCliArgs(cli, taskName, sources) + if err != nil { + return terror.Annotate(err, "while removing task command line arguments") + } + } else { + err := ha.PutTaskCliArgs(cli, taskName, sources, *cliArgs) + if err != nil { + return terror.Annotate(err, "while putting task command line arguments") + } + } + return nil +} + // nolint:unparam func (s *Server) convertTaskConfig(ctx context.Context, req openapi.ConverterTaskRequest) (*openapi.Task, *config.TaskConfig, error) { if req.TaskConfigFile != nil { diff --git a/dm/dm/master/openapi_controller_test.go b/dm/dm/master/openapi_controller_test.go index cfb314027fc..7925366b6fc 100644 --- a/dm/dm/master/openapi_controller_test.go +++ b/dm/dm/master/openapi_controller_test.go @@ -22,6 +22,8 @@ import ( "context" "testing" + "github.com/pingcap/tiflow/dm/pkg/ha" + "github.com/pingcap/failpoint" "github.com/pingcap/tiflow/dm/checker" "github.com/pingcap/tiflow/dm/dm/config" @@ -372,6 +374,24 @@ func (s *OpenAPIControllerSuite) TestTaskController() { // stop success s.Nil(server.stopTask(ctx, s.testTask.Name, openapi.StopTaskRequest{})) s.Equal(server.scheduler.GetExpectSubTaskStage(s.testTask.Name, s.testSource.SourceName).Expect, pb.Stage_Stopped) + + // start with cli args + startTime := "2022-05-05 12:12:12" + safeModeTimeDuration := "10s" + req = openapi.StartTaskRequest{ + StartTime: &startTime, + SafeModeTimeDuration: &safeModeTimeDuration, + } + s.Nil(server.startTask(ctx, s.testTask.Name, req)) + taskCliConf, err := ha.GetTaskCliArgs(server.etcdClient, s.testTask.Name, s.testSource.SourceName) + s.Nil(err) + s.NotNil(taskCliConf) + s.Equal(startTime, taskCliConf.StartTime) + s.Equal(safeModeTimeDuration, taskCliConf.SafeModeDuration) + + // stop success + s.Nil(server.stopTask(ctx, s.testTask.Name, openapi.StopTaskRequest{})) + s.Equal(server.scheduler.GetExpectSubTaskStage(s.testTask.Name, s.testSource.SourceName).Expect, pb.Stage_Stopped) } // delete diff --git a/dm/errors.toml b/dm/errors.toml index ed7c1214ae4..5ac789ecfc5 100644 --- a/dm/errors.toml +++ b/dm/errors.toml @@ -929,7 +929,7 @@ workaround = "Please check the `target-database` config in task configuration fi tags = ["internal", "medium"] [error.DM-config-20022] -message = "mysql-instance(%d) must set meta for task-mode %s" +message = "mysql-instance(%s) must set meta for task-mode %s" description = "" workaround = "Please check the `meta` config in task configuration file." tags = ["internal", "medium"] diff --git a/dm/openapi/gen.server.go b/dm/openapi/gen.server.go index dd4cc1725fe..bf40d67a7d9 100644 --- a/dm/openapi/gen.server.go +++ b/dm/openapi/gen.server.go @@ -1248,108 +1248,109 @@ func RegisterHandlersWithOptions(router *gin.Engine, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+w9XXPbOJJ/Bae7h5kpyZJsx0l8tQ9J7Mn6zvko21NzW1M5BSJBCWsSYADQHm3K/30L", - "HyRBEiAp23KssfdhxxFBoNHo7240vw8CmqSUICL44PD7gAdLlED155sYMfEBErhA7IKmNKaLlfw9ZTRF", - "TGCkRi0pF/K/6E+YpDEaHA6muy93JjuTnelgOBCrVP7EBcNkMbgZDlLKqsNfT17vFeMwEWiB2ODmZjhg", - "6FuGGQoHh3/oRczLX4rRdP5PFAg567s44wKxD1D+fxNGGIbq1xDxgOFUYEoGh+pXxDmgERBLBIKMMUQE", - "SNQkgNAQDYaubR2+2j1w7g3G+Ao116EkxgQBLqDIzGqYm2XsFQTLUDHrnNIYQSKnjREMkQN+zO2Z1B7M", - "0B6TEpig6rHpaRwbq52FejPfbAHdUCO55XD8JAQloc0STWkzYY37L4aiweHgP8clkY4NhY6d5HkzHCwY", - "jCCBved5r8fbU2hUFDPMYqxpHAuU8K75NBHa0xmMQMag+nfKaILEEmW8N5Cfi1fsia8pu7w1nL+rl/1w", - "3viPUr/6w/hsTjMSzjjNWIBmOSFX11RDgB4C5JCC7zTOmssmK/4tHk3aFhRw4V9KPuxcRI11rdBkRz1F", - "f3aUqK9C6kKUkz8puUJM0izkl2foW4Y0FVXPVkB+2UVScgJFSJBfzgJKIryYRTh2IE0/BPIhwASsYBKD", - "iLIECrAUIuWH43FIA76TYrIIYLoT0GT8r+VY4HA+5gLOYzSWi4z0PBmDct6RnG4UZXG840Rb1855SglH", - "f8mt2xSjtuOA1EkbDEGBzhUFeUlDE1gXhvQkltjy0fyom+jNin6I74mUXZhzLXqEuTyYMxTDlbVsTQ4G", - "8g8gKOCCpgACJocDZsYPa1BaWCoEe7c8/wgTdCpHOwn+KEvSc2WHNMEr7ZMwS1KQEdyESS4bI4HCmSJE", - "9Zum3cHhIKTZPEbl2ZEsmUtbbjhAXOAECjQTVMB4xuh13zcjTDBfonA2Xwm09ktrLKQhc+wKE3GwP+i0", - "UCvvD5uIamylDqYbSy5iOybr0RpkopPY1NPZHJOYLmYLgUMnfTCByQK8vzg5ypV5lnLBEEyAfrWi7NBr", - "OI2C3d0RCiavRtMpej2a78JgNNnd34XBdDqZTPYOp6OXr/ZfD4YDksWx3FfNZC1VZAVEt9YvQJTyrNT6", - "7WBqxT/HZGci/7fbH5YQG2snglksaWVnrB/oJaqwSTBCzFAgKFuB6yViSIGmzyWmC4C5FAySnnpAsAnp", - "cMwYZb9jsfyAOHfaOpJklL4BSI5tkJH6dRZIs6fxrnoGAm0S1blpaF5N+ML3ZmKA6tIN5URDGx4XJ71H", - "wli0JySifgMg0INmLrYwzwCWx1ZIjcwnNqSk6Wfy192m+j4toNr3ph0Seez+HYZQwN6eQ9Xbdjg4SoAp", - "TdtDaEpOkau3b0LT7/1vwrgyG96Etn3uEfrSmNo82NpguFfAjQ2yYfClDXePOC9M/A2D/AEvmDJh2QIJ", - "fo/AVyZ+iJ3cL+Vk83LOh4D+Qirgc8GyQGQM+XehAZwFyvGY8W9x1al5d3b85uIYXLx5e3oMvorpV/DT", - "Vxx+BZiIn6bTn8HHTxfg42+np+DNbxefZicf350dfzj+eDH8fHby4c3ZP8D/Hv9Dv/EzGP9y8R9/GLmP", - "whkmIfrzC3h3+tv5xfHZ8RH4ZfwzOP74/uTj8d9OCKFHb8HR8a9vfju9AO/+/ubs/Pjib5mIXiXzffDu", - "0+npm4vj/N/SrHKFJczWmp5aOHcGSpSx6xiufp/28EyL1/O5LKw6j6oWvLv38PTeZDK5c3j6lMKw2+2K", - "KQzdbleLF+S3MxIkoDGXLaYot2o9Lyz+Jj4YXTDEufOh9lP6w1TDWsMhsuezlq5uxQG4C+W1KOxd6cIX", - "Lu9FQ692D6ad2DBU30VKn5QFjtoDVsESBZczhrhyS+oUlzI0UiOAGWF7Q+VDzEEKOUfhDnCz+l2CKMMq", - "jB07rUviTqdX+ykIKBnidXqjOOPLigenna3qrL8zLBBXvprel1xABbLlDlKKiQBc/gIFOPoAAkg0J2MB", - "YCQ9A4YKv1S+loffGikZ/i2eBZQIRBx7499isKIZuIZEWDusnJ1D04CvwbRUNbk2kOpmCL4Gu/5He+5H", - "d9Av/+1UMCsSNDf7WxrCHOc0FTjBXOAA8CVkoUSjlABSe4NrLJY64m6OhpJ4BTKOQulhEwCNowpoEGSM", - "A0y8cx4dnYKk4pwWR1MPPlrn5CJcR65mE1nTu6ulzxlzOfllRCKQ+89SkNIYBytQiTg3ff8/U8yMXZfz", - "06TOTGqQjiAIrOMzxXK2/5yrEE8cxFJz8k92pQ2/Yt29g0lj6YslAvlgyUEpYpiGOIBxvAJG5EXNkIze", - "VjgEZnJwBeMMHQK1hCQojgJKQn476BlKICYznsIAVXYwfVGH/wMmOMkSEDGEQIj5JVBvKRjev73N8jc+", - "mrjXOPYDxu264nSVNVMU4GhlgOfZ3IrORZSBBtg74CQChAqg38SSJlTOXYoqAShB4BrHMZgjJYB2wLmC", - "1OR2DsEuRC8P9vf2R9HL19FoOkWvRvMQ7ebhUGlovtJbmXYHAGuc3sSxi9/Vsb5TTNzEh9JoOjWVM2WT", - "xVXkeaYflhahpcOe48hbFUe+8VFJt7dii+0qlZjqidL1qE5Rw2GeCNVsohVLidSfalidDsH09cvXP7uY", - "vbKuh/hcNHcHYmsnLjcIGnF5FYQE6P4BCKAIlrMsnSVFRVQViOultFCYFOJqLMhSbUwVp2O5Xz42d8rV", - "9eiz3PfOmGdzNaXLTHSXXuRI1FRZme4sI0S+3CU5q8TqJCJ7u64T9iE9B9slis+VuVqkY5p8ps1ZJXtU", - "emdYBsm6ozC1wNg5CjKGxaq5jDKiTZUM53HVwtPqLcIoDgvNtsRhiIg2rhdIFE6NPVFlEhAxmqghyvaK", - "pJ3TFEs19xUxMYNxTK9ROAtIE+x3NEkoAR+NZD4/PwXyHRzhAOrgQYGsTuRwHs8C6He8rIm1qMpH2tTm", - "pFk5sdyJd+pfrenkPj4ffzDWwvj/Xkxe51Uhta11r3qJVv5F35XryVNJGb6SW7tEq6IkxVq8Y726Z1TF", - "pQMHTQCd3GGcsveMZqkjbBzGzVK3zoOOMONiFtNAaxnXK9IbReF60wodTXcNzcj6EzaCJWr2YbnnxkYK", - "sK0FnUgtqnRqokb/7rb1KnZJBGPeCI8UmkR54VoCSLdJvV4R8eb1pjYxZmWpLnutR6WZrY1I6c1lUkAp", - "qcy1zHGpdy8IUQyvqEOb6d+Lur4CVzWzz8WJuYvvwjYwNZHuwkdnBAByfk1Z6J2xGFCdcm//xUEfSzSP", - "MLjnlg+teff2JgcubzbNAwqtpaxqUGmqFP5I20u26yIZ1dJorTmjfJx8p61e1KoU7V0Vqq2O9YpuO9Of", - "kF/2L+q4gPyyLOkYDjLusvXM3uTDxv4YpaJntd3MEaE2S1ZZOP9XixRqMXyskl2/4aNHjfpZPzbKfesV", - "FqSrnqW7KEUbRFzF/aRJdM2oy/bMaZ4XwHTSfEkqd6BfhtIYB9BDx7VyzGbUzFQzG2s7XgFd8mzC4A6Z", - "uGYdZ05ZNiBO2pFueWtlJ0MJvUKzBOm8cm9Not9TcWVlys4hV5ZQSK+J8Yfyn92h+3IfvRi3xgVOP/xc", - "0LR1sxtZtJJkb1peWZL2pF6r1HSN6q7ejBRTGPaExMq+Wpm8mtcK+WXOPU1B35/z/J5rlxw/VwONEdpz", - "Z+crEpQ7U6lj987kI6Bgs9lUJV+GLruVIU7jKxTOlNlJg8uZJz/cKjvyWxBO1LjTn36BkKPS7NMpH0p0", - "tASu5K7daXbj1Ot5HZudS0xgspBYcS1hp5KulzhYFlEezEH+8lrOaSOU1jPo5dA7ASJiJtK+1QMmqzGb", - "oyUmoRVH6vNu4fU4JKV81rqjygj/jnSxALrKLy72gMvUaPfGgcUHC+mJtp25HlA7dsgQyMgon8U++la2", - "rri/nS6ijQh7k5VTH/aLdFWPx3kYdT5w4cnySW2m8pGVi5lVzv+uATJf3VGT0y5MOUNTePrERIRjiT+W", - "aS8ZhiGWb8H4c2V0l9x/i8kpXfyqJjuTc7nUMiJLSAI001dJZ3nF2RKSBeosYLDsHG2YA56l0nxXeS6V", - "D9c3VMMwBmmcLTDpc4MULwhlaKYyp5IYCvTXbqmqYSBlyORY1TDnaV0hxnVEo1swIgENGqqpkzAZKeuv", - "jgSHJae2zwVleUmBNwtRTuotDPKbEzY18ku3z0LJLMyUjS4csy3ptTy8JSShDhhGMQ4ECtVOlLuVJToL", - "mMY6vppX52vkW/xliVkpZGaJs1Rf0sQ1XKnsAKVSFkGBpFqzFksR56aIYjAclBUV7sW0Wu/n6ytrSL1g", - "Ofy38bU7C0aVz5roqtiCkesnKRnGjAFqzLB/xa0SYqbstsbctQDiGrjR9btHUMC30h/Jowbuo8whN6Ut", - "+elFWRzLjZCAoQQRXRALY1VkWRIsjOO+hlsJQoe0qhF7ff/OU6kTkFtfOGSpK+QukGJ4OTEHUORpyBhd", - "obgh642QU9rV4brIn3O72iP/KmMqqAVhEveRdQYGU1jcLAtLoRCIqYIMrZP8wPiGl3D9/xGjaTdUN54T", - "+DWLY0Pvknl9t1/z5BCNgKTEgr8kFXHHrUPCMReIBI4UlpJRRDAag1xsYWLsMJWV0jU8lEmBGakbSMVs", - "AHKeMUmr1bPJBHWhQE7nKSYRlEnvNcSsKfZ3xvn6MyOwGzPrATOxZAiG1RKq/bomUwjTL0j8BZQYc9Np", - "w+LEO/P0wDm1fqNzah8FnJCArUcBlhDyEIBUbLM5FEG1WnPaLPKy55Im6JJRgv9VLKXmAOhPFGTqJ8kP", - "3zJIBFZLuSu00rgn+uobuTUOqxc13NZFyTLqmkgDZ0ZiljZSZ9rYvCHyvI9luPiq/ZXkXmMJ80bfJdzR", - "QrNeDeA6OLXFfCrD72EUNlyrf8Eve7sXpU3TDKzVvN1yhcleFEx2D/ZGu6+Cl6PpFL0cwYMXe6ODYDJ/", - "tR++eB3tTQ6no5eT/en+7t5w8mL/5X64F1jDX+292B3tTvbC+e7+QRjuhYfT0fTlxNnnoloUZfWtUA/K", - "6jTfmymtImjfGR7YTCC7JbTsO/yKlekBZcRQDKXuaK9+laKzMFoCc8ZdllxdW95oi2zteeoyt2pxe5Fc", - "31Fvs9ai5K7ohA2H9xjyGGlunZ4LmqYqelCW8fxqbos4/Qunre2vPNNGvaB2fN828XlPn7+mPdVDNUFO", - "vw6RIR/3S1zx1oR9T7q0fWRP/GQIrnEcBpCFeWCg6vzOR7/cMSreSNz5ouWirDloOmE9YBVOWFuTTpa6", - "8OkJ4dHDJfXc52GEFHFdZ2yiNPmOee1YprfEYM8FfBq5hp7+nVkcvmsLSsswTTtOH1WZxWbKKm5T7bCh", - "UgBn8r/AiffUUZJK/vCmMukVYtcMC7RW1rZ4S1vbwqxS/NF9ladctxt032W7COJY9Xnhl834VEs5gfNG", - "XSFOu1s45QKsnNQpu+pKJQsCxLkH3PWK05pzDZvYcAGl73fda1ep/mJIL/7ADaJq7VfaUqUt7oa/rqJ5", - "0OWK3os85sYOB7n2EtTUevC2blRdid5b1IF0VX7UehXe/21eb7e9jV7nvVGhHyGFcXxEA0fA7ugD+JQi", - "8ubzCTj69E6KXBYPDgddjeJGUnmOtEmLKTF947R/EVFF4liojTcWyJMwh4MDiUCVnUgRgSkeHA721E9S", - "4oulgnYMUzy+mo5NU4JxPr2xl4p+QSehWuvN55Nqzx2VpNSSVc23O5mYiF9evQxTHSqW2/gn14W7pR3V", - "2tjT3d1HYb2mFrUgU4fIsySBbDU4lHsARXcfElHAs2AJIAeVlj8CLrjVjmfwRdVB+navhU8dAYoN39Jw", - "dW97bzYPamzaLAvmct2bR3wOmcJZ5Sh2nIi/GTboUSeYeV+SLFslPQxhOloztaFlONi/RzAa7b4cS2t1", - "3sIYVhfXXHGtczDj7/oP5RHeaPkXI20HOk7qUxTFmCCNto8625RCBhOkT/mPRvrLAi/3yVUPBCiWg1wR", - "DCwYBrYY16lvV3zT3yz5S4Nw9h12+CM7UarxWuvJ2+sgc4OhJ4eVfbwehsMcfcO2jMOsXsJrcZg5mPF3", - "Y4WtxWHGeuzBYTZ4fg6zYHjaHFbtDN16kGGykwPn5Kz3SBzR4H/OP330sFIVLDlXcXmtSW4hDYBaroQq", - "pEENImOjtoDz94sPp73AkQM7wFkKnSD3gaOdvG7RU3bf6yJmyV/5JSZ1Hba4F6Bo+luG2MoiaiyWs2KE", - "g4jdpVM3Q8cXAlaAIZEx3Z9El2mNTGuCvL7eBULlRv46MHzZrPR1NDx0cIp9azTOa9JrdFAfUtJD7uMr", - "H437zt/uYL0pY9vRJHt9g3t6b/AUMZFHr+d0dzcASZiXJkJA0LV96q4Db8qA8Xcrs9Ct5Y7Uw4IoWmXC", - "IqZz1SMmI/hbVr3q7Fd41URHL4XnvWrWFBgR1ZeWaJpDAmNu+rHkl+1VQMeUU7hEh5rjjjJjCxSvpgMA", - "u2hq2EeHbCOtPIxO26Q+aZFnRRvcfSctGsxTASL15Y2mfmkjiK4wztbQxJfN6D1XGP+mGgiV4N78GNJ4", - "ZHLIRLHgXXXbONTfmlBBcL/ZY75IsV0k2uUzPDrdopF8D4dadltoOVP94YfnI93kkRZm6F1PVLlk6zHr", - "Wd507WmqE9dHdG6MPtlWyVB2vYoyovsm5peu7ofA1hAcT5y8HJ/N2VbqMkJq48RV9HNpoa2yYejTJa1m", - "09T+ZvDjpjRFAZVej+vTkvVJ2R4utu6M1ydYuwHS8feV2ayDW+0GuCUJqrx1ji5e9QVn+5LH+Lv+o4zg", - "9SAWVfP9+Ghl2FLg61m+3HvP5Z31vxul0uqN/O0iUl3/fHsaLbqK9JFgRS+px6MNWy/OPEguqPYtoC0h", - "H/uDxfYHvO9uYQkGCY/MV5T95tWFGfbUY43Ncta/iomVE0IhqiiAusm/rhXooC6d4umSTPmn0DoJSNI8", - "5JcPmf0296bmK71y3ubJtWb+rK/CKtpqta3q4I/6svVOa8O1wtOWztywqG188c5BhArJsekY+XgEbQFV", - "Se66mr5Pev9C9+nZXHLfvi7wI1P7rs8/bVGev/j4UfWE6+JsHOQfZucd6tF8wX2T5+/6Or5j3zjSNIw5", - "wCTNhG4YbGSpbp6e70q3ziw/v64bb1MGrnCAwBViHG6UiNyfvd8CMrpQBVIKy8R0HzU90mkEYL3xfAOp", - "Oz0oL7871k+l5rfDHqCedctFe3E5704y/qK82bcJXjd3un6cePcB8EjleeVk12GusW4y0yHcT9SgBzr3", - "+h3V9clgd0PwbI98Nq2Dbk8W31WLvHVq+GrUsZZ3bHfpc7jFBSw9nWJfe7+trpvz36yuC/DeynJ7jmny", - "5AR7U1+3Hbm3QK68Y/186FtTmtb33Bvy+3ZS+7FSRFuxtYIBXSECcKS61AOezXO3jxW9ip7LrX2efg81", - "sTV08QCx0h8hnWpO5L6vM15LUbX/9LtKqh8zAWy0ivpuAcbJUw8wFtXVPQOMlsry5OfyHnx5f80+4aBK", - "306+NYLswYsjnDkW3SXadCce+Ioefuk/o24k3T6hGvPLw+fEm9SydZlxlauzqysgCU03WvMDo5kwd9Fw", - "5WLx7bmydy1ZUUX2diVx/YaEt8ugPxGmfK5ua6Nvd4nbnal4zZK3otjtmaSfi/C2lpeclXj3zEryvXmM", - "1gxJzGN0LlgWiIw989Rj46mhv6OtD+U5BfTGuftbUdsfvq9wHrdIfN3gzDOHPHPI9Mc4S1Xi235nqZUN", - "/VGyIjzzzIprL/5UGPH+Q5RWULDOh3+tWmzNcWuqzXarVcDOOpfiu9ZPLPLd+J73tt7HVYd8y+Bzv5tF", - "1ocMt1DYFy3Nt722fksvMZlrFZp61qNOmnYKL/2Z+icnu6pf599e0UVTv+RSHx9hV/mJVpvPr2i2E9IE", - "YqJazw8kqs0Eblkw6Op2H9Kgd4t709N+/C3DweVISeCRLksdlV3BKjJm4LLM1LY3C9U1FstRmFjwqGWb", - "0ORdYItx+Q83X27+HQAA//9o+0ohqLUAAA==", + "H4sIAAAAAAAC/+x9bXPbOJLwX8Gj5z7sTEmWZDtO4qv9kMSerO+cl4o9tbe1lWMgEpSwJgEGAO3Rpvzf", + "r/BCEiQBkrItx5pkP+w4Igh0N/odjea3UUjTjBJEBB8dfxvxcIVSqP58lSAm3kECl4hd0owmdLmWv2eM", + "ZogJjNSoFeVC/hf9AdMsQaPj0Xz/+d5sb7Y3H41HYp3Jn7hgmCxHt+NRRll9+MvZy4NyHCYCLREb3d6O", + "Rwx9zTFD0ej4n3oR8/LncjRd/AuFQs76Jsm5QOwdlP/fhhFGkfo1QjxkOBOYktGx+hVxDmgMxAqBMGcM", + "EQFSNQkgNEKjsQut4xf7R07cYIKvUXsdShJMEOACityshrlZxl5BsByVsy4oTRAkctoEwQg54Mfcnknh", + "YIYOmJTAFNW3TU/jQKyxF+rNAtkSurEmcsfm+FkISkYLUs1pgbDG/QdD8eh49P+nFZNODYdOnex5Ox4t", + "GYwhgYPneavH21NoUpQzBAnWPI4FSnnffJoJ7ekMRSBjUP07YzRFYoVyPhjIj+Ur9sQ3lF3dGc6/q5f9", + "cN76t1K/+t3kbEFzEgWc5ixEQcHI9TXVEKCHADmklDtNs/ay6Zp/TSazrgUFXPqXkg97F1FjXSu0xVFP", + "MVwcJenrkLoI5ZRPSq4RkzwL+dUn9DVHmovqeysgv+pjKTmBYiTIr4KQkhgvgxgnDqLph0A+BJiANUwT", + "EFOWQgFWQmT8eDqNaMj3MkyWIcz2QppO/72aChwtplzARYKmcpGJnidnUM47kdNN4jxJ9pxk68OcZ5Rw", + "9KdE3eYYhY4DUidvMAQFulAc5GUNzWB9FNKTWGrLx/OTfqY3K/ohfiBWdlHOtegJ5nJjPqEErq1lG3ow", + "lH8AQQEXNAMQMDkcMDN+3IDSolKp2Pv1+XuYonM52snwJ3maXSg/pA1e5Z9EeZqBnOA2THLZBAkUBYoR", + "1W+ad0fHo4jmiwRVe0fydCF9ufEIcYFTKFAgqIBJwOjN0DdjTDBfoShYrAXa+KUNFtKQObDCRBwdjno9", + "1Nr74zahWqg0wXRTycVsp2QzXoNM9DKbehosMEnoMlgKHDn5gwlMluDt5dlJYczzjAuGYAr0qzVjh17C", + "eRzu709QOHsxmc/Ry8liH4aT2f7hPgzn89lsdnA8nzx/cfhyNB6RPEkkXg2XtTKRNRDdVr8EUeqzyup3", + "g6kN/wKTvZn83/5wWCJsvJ0Y5onklb2pfqCXqMMmwYgwQ6GgbA1uVoghBZrel4QuAeZSMUh+GgDBNrTD", + "KWOU/R2L1TvEudPXkSyj7A1AcmyLjdSvQSjdnta76hkItUvUlKaxeTXlS9+bqQGqzzZUE41teFyS9BYJ", + "49GekZj6HYBQDwpcYmGeASy3rdQauU9tSE0zzOVvhk1NPC2gunHTAYncdj+GERRwcORQj7YdAY5SYMrS", + "DlCaUlLk6t1IaP59eCRMKLNlJLTv84DQV87U9sHWDsODAm58kC2DL324B6R56eJvGeR3eMmUC8uWSPAH", + "BL428WNg8rCcky+qOR8D+ktpgC8Ey0ORM+THQgMYhCrwCPjXpB7UvPl0+uryFFy+en1+Cr6I+Rfwly84", + "+gIwEX+Zz38B7z9cgve/n5+DV79ffgjO3r/5dPru9P3l+OOns3evPv0D/PfpP/Qbv4Dpr5f/759G76Mo", + "wCRCf3wGb85/v7g8/XR6An6d/gJO3789e3/61zNC6MlrcHL626vfzy/Bm7+9+nRxevnXXMQv0sUhePPh", + "/PzV5Wnxb+lWudISBrV2pBYtnIkS5ew6hqvf5wMi0/L1Yi6Lqs6taiTvHjw9fTCbze6dnj6nMOoPuxIK", + "I3fY1REF+f2MFAlo3GVLKCpUreelx9+mB6NLhjh3PtRxynCYGlRrBUT2fNbSdVQcgLtI3sjC3pcvfOny", + "QTz0Yv9o3ksNw/V9rPRBeeCoO2EVrlB4FTDEVVjS5LiMoYkaAcwIOxqqHmIOMsg5ivaAW9Tvk0QZ12Hs", + "wbSpiXuDXh2nIKB0iDfojZOcr2oRnA626rP+nWGBuIrVNF5yAZXIlhhkFBMBuPwFCnDyDoSQaEnGAsBY", + "RgYMlXGpfK1Iv7WOZPjXJAgpEYg4cONfE7CmObiBRFgY1vbOYWnAl3BemZrCGkhzMwZfwn3/owP3o3vY", + "l/90Gpg1CdvI/p5FsKA5zQROMRc4BHwFWSTJKDWAtN7gBouVzribraEkWYOco0hG2ARAE6gCGoY54wAT", + "75wnJ+cgrQWn5dY0k4/WPrkY13FWs41T0/ubpY85cwX5VUYilPjnGchogsM1qGWc27H/Hxlmxq8r5GnW", + "FCY1SGcQBNb5mXI5O34uTIgnD2KZOfknu9aOX7nuwdGstfTlCoFisJSgDDFMIxzCJFkDo/LidkpGoxWN", + "gZkcXMMkR8dALSEZiqOQkojfDXqGUohJwDMYohoG82dN+N9hgtM8BTFDCESYXwH1loLh7eu7LH/r44kH", + "zWM/Yt6uL09XWzNDIY7XBnieL6zsXEwZaIG9B85iQKgA+k0seUKduUtVJQAlCNzgJAELpBTQHrhQkJqz", + "nWOwD9Hzo8ODw0n8/GU8mc/Ri8kiQvtFOlQ6mi80KvP+BGBD0ts0dsm72tY3Sojb9FAWTR9NFULZFnGV", + "eQ70w8ojtGzYzzzyTuWRb31c0h+t2Gq7ziWmeqIKPepTNGhYHIRqMdGGpSLqXxpUnY/B/OXzl7+4hL22", + "rof5XDx3D2brZi43CJpwRRWEBOjhAQihCFdBngVpWRFVB+JmJT0UJpW4GgvyTDtT5e5Y4ZdPzJ16dTP+", + "rPDem/J8oaZ0uYnu0ouCiJora9N9ygmRL/dpzjqzOpnIRte1wz6iF2C7VPGFclfL45i2nGl3Vukedbwz", + "rpJk/VmYRmLsAoU5w2LdXkY50aZKhvOk7uFp8xZjlESlZVvhKEJEO9dLJMqgxp6oNgmIGU3VEOV7xdLP", + "aaulRviKmAhgktAbFAUhaYP9hqYpJeC90cwXF+dAvoNjHEKdPCiJ1UsczpMghP7Ay5pYq6pipM1tTp6V", + "E0tMvFP/Zk0n8fh4+s54C9P/eTZ7WVSFNFDrX/UKrf2LvqnWk7uSMXwtUbtC67IkxVq8Z71mZFSnpYMG", + "bQCd0mGCsreM5pkjbRwl7VK33o2OMeMiSGiorYzrFRmNomizaYXOpruG5mTzCVvJEjX7uMK5hUgJtrWg", + "k6hllU5D1ejf3b5ezS+JYcJb6ZHSkqgoXGsAGTap12sq3rzetibGrazM5aD1qHSztRMpo7lcKiillbnW", + "OS7z7gUhTuA1dVgz/XtZ11fSquH2uSSxCPFd1AamJtJd+OjMAEDObyiLvDOWA+pTHhw+OxriiRYZBvfc", + "8qE178HB7MgVzWZFQqGzlFUNqlyVMh7peskOXaSgWhat88yoGCff6aoXtSpFB1eFaq9js6Lb3uNPyK+G", + "F3VcQn5VlXSMRzl3+XoGN/mwhR+jVAystgscGWqzZF2Ei391aKEOx8cq2fU7PnrUZJj3Y5Pct17pQbrq", + "WfqLUrRDxFXeT7pEN4y6fM+C53kJTC/PV6xyD/5lKEtwCD183CjHbGfNTDWz8baTNdAlzyYN7tCJG9Zx", + "FpxlA+LkHRmWd1Z2MpTSaxSkSJ8rD7Yk+j2VV1au7AJy5QlF9IaYeKj42Z26hzEKUhqhQOAUBVGRI21H", + "RzhFoHgszYp8s8g7W3p7xp0apyLXIP3QEDats5hQQDpgg/zKlAuqATZA+7PZ0WQ2n8z2wfzZ8ezwePZs", + "WIn1haBZ55bdHycJLM3FYKrfQKzjFo0vzeqkf8YHYlarR2g7qXmaDRR0qyp3g0K4wTonoTAaCIl1UG0d", + "ejrYpDij7+DQPiXlD/L7TN6FGmj89YGYXaxJWGGmTtndmMlHQMFmc4U6pxq7XHyGOE2uURQoD52GV4Hn", + "KL1TzRYXRpykcZ8U+3VnQUqDp1OVVuToyPFJrN0VCSb/oed1ILuQlMBkKaniWsI+dbtZ4XBVJsQwB8XL", + "G8XxrazjwPygw0SHiIhAZEMLLcwBULBAK0wiK+U25N0yQHQYFfmsE6PaCD9Guq4CXRd3PAfAZcrZB9PA", + "koOlDNq79lwPaGw7ZAjkZFLMYm99p1jXMgW90bRNCBvJ2q6PhyUF69vj3IymHLjoZIXvtlD52MolzKo8", + "4r65RF+JVlvSLk3lR1t5+tREjBNJP5brhAKMIizfgsnH2ug+vf8ak3O6/E1N9knO5TLLiKwgCVGgb90G", + "RXHeCpIl6q31sFxCHcMAnmcy0lFHgqp0QF/mjaIEZEm+xGTIZVu8JJShQB0yS2Yoyd+40KuGgYwhcxyt", + "hjl36xoxrpM//YoRCWjIUD9litKJcpSbRHA4vQp9Ligrqi+8BzbVpN4aKr87YXMjv3KHd5QEUa7CGeGY", + "bUVv5OatIIl0bjVOcChQpDBRkWme6gPTLNGp6OIigya+JV+WmpVKRrn37uOOG7hWBymUSl0EBZJmzVos", + "Q5ybepPReFQVn7gX02Z9WFpEeUPqBSs3cpe0RG9trQrvU11AXApycyelwJgxQI0ZDy9OVkrMVCg3hLuR", + "a92ANrrU+QQK+FqGbkWCxb2VBeRFNGZ2L86TRCJCQoZSRHTtMExUPWrFsDBJhjpuFQg92qrB7E38nbvS", + "ZCC3vXDoUtfphEBK4OXEHEBRnNgm6BolLV1vlJyyro7QRf5c+NUe/VcbUyMtiNJkiK4zMJga7HYFXQaF", + "QEzVrmib5AfGN7yC639PmIod+zP6zh34LU8Sw+9SeH0Xha1cgeTEUr4kF3HHBU3CMReIhI7TPqWjiGA0", + "AYXawsT4YeoAT5c7USYVZqwua5WzAch5ziSv1vcmF9RFAjmdp+5GUCaj1wizttrfmxbrB0Zht2bWAwKx", + "YghG9Wqzw6YlUwTTL0j6hZQYd9Ppw+LUO/P8yDm1fqN3ah8HnJGQbcYBlhLyMIA0bMECirBe2Dpv18PZ", + "c0kXdMUowf8ul1JzAPQHCnP1k5SHrzkkAqul3MVsWTKQfE1E7kzD+p0Wt3dRiYy6UdOimdGYlY/Ue8Ju", + "3hDFEZnluPguRijNvcES5o2hS7gTq2a9BsBNcBqL+UyGP8IofbjO+IJfDQ4vKp+mnVhrRLvVCrODOJzt", + "Hx1M9l+EzyfzOXo+gUfPDiZH4Wzx4jB69jI+mB3PJ89nh/PD/YPx7Nnh88PoILSGvzh4tj/Znx1Ei/3D", + "oyg6iI7nk/nzmbMlSL1+zGrxoR5UhXy+NzNaJ9ChMz2wnZx/Rxbet/k1L9MDyoShBErb0V0oLFVn6bSE", + "Zo/7PLmmtbzVHtnG8zR1bt3j9hK5idFgt9bi5L7shA2HdxuKHGnhnV4ImmUqe1BVPP1mLtY44wunr+0v", + "0tNOvaD2UYjt4vOBMX/DeqqHaoKCfx0qQz4edsbHO2sbBvKlHSN78idjcIOTKIQsKhID9eB3Mfn1nlnx", + "1hmnL1suqvKMdhA2AFbhhLXzfM4yFz47ITx2uOKeh9yMiCKuS7JNlqbAmDe2ZX5HCg5cwGeRG+QZ3sTG", + "Ebt2kLRK03TT9ElVpGynAuUuhSFbqppw1kmUNPHuOkozKR/e81J6jdgNwwJtdMBdvqW9bWFWKf/ov/VU", + "rdsPuu9eYgxxolri8Kt2fqqj8sJ5+bBUp/3drgoFVk3q1F1No5KHIeLcA+5mdXztucZtariA0lfhHrQB", + "13A1pBd/5F5ajU41XUelHeGGvwSlvdHVit47T+ZyEweF9RLUlMXwrsZdfQe9dyiZ6SuSabR1fPiLz97G", + "hFu9+XyrUj9CKuPkhIaOhN3JO/AhQ+TVxzNw8uGNVLksGR2P+nrqTaTxnGiXFlNiWuzp+CKmisWxUIi3", + "FigOYY5HR5KA6nQiQwRmeHQ8OlA/SY0vVgraKczw9Ho+Nf0bpsX0xl8qWyudRWqtVx/P6u2J1CGl1qxq", + "vv3ZzGT8ikJvmOlUsUTjX1wXwlR+VGcPVHcjJEX1hlnUikxtIs/TFLL16FjiAMpGSCSmgOfhCkAOat2R", + "BFxyq3PR6LMqGfVhr5VPkwBKDF/TaP1guLf7LLWQNsuChVz39gnvQ65oVtuKPSfhb8ctftQHzHwoS1Zd", + "pR6HMR1drLrIMh4dPiAYrc5ojqW1Oe8QDKvhbWG4NtmY6Tf9h4oIb7X+S5D2Ax079SGOE0yQJtt7fdqU", + "QQZTpHf5n63jLwu8IiZX7SKgWI0KQzCyYBjZalwffbvym/6+0p9bjHPo8MOf2I5STddG++JBG1k4DAMl", + "rGp59jgS5mixtmMSZrVd3kjCzMZMvxkvbCMJM97jAAmzwfNLmAXDjy1h9SbanRsZpXsFcE7JeovECQ3/", + "6+LDe48o1cGSc5X3/NrsFtEQqOUqqCIaNiAyPmoHOH+7fHc+CBw5sAecldAH5D5wdJDXr3qqRoV9zCzl", + "q7jvpW4Ol1coFE9/zRFbW0yNxSooRziY2F06dTt2fExhDRgSOdOtXHSZ1sR0cSiuIrhAqDUv2ASGz9vV", + "vo7ekA5JsS/YJkUH1wYfNIdU/FDE+CpG4779t5t9b8vZdvQT39zhnj8YPGVO5MnbOd0ID0ASFaWJEBB0", + "Y++6a8PbOmD6zTpZ6LdyJ+phyRSdOmGZ0IVqp5MT/DWv3wr3G7z6Qccgg+e9lddWGDHV97toVkACE25a", + "1xR9CVRCx5RTuFSHmuOeOmMHDK/mAwD7eGo8xIbsIq88jk3bpj3p0Gdlx+BDJy8aylMBYvWRkrZ96WKI", + "vjTOzvDE5+3YPVca/7aeCJXg3n4f1nhieshkseB9bds00p/lUElwv9tjPt6xWyzaFzM8OduiifwAm1o1", + "pujYU/2NjJ9bus0tLd3Q++6oCsk2E9ZPRX+6H9OcuL43dGvsya5qhqpBWJwT3WKyuHT1MAy2geL4wdnL", + "8YWhXeUuo6S2zlxl65sO3qp6q/64rNXuLzvcDX7anKY4oNYWc3Nesr6+OyDE1k0EhyRrt8A6/hY82w1w", + "640Td+SAqugypItXfcnZoewx/ab/qDJ4A5hF1Xw/PV4ZdxT4epavcB+4vLP+d6tcWr+Rv1tMquuf786j", + "ZVeRIRqsbLv1dKxh58WZRzkLanw2aUfYx/62s/2t8/t7WIJBwmPzwWm/e3Vphv3oucZ2OeufxcUqGKFU", + "VRRA/T0EXSvQw136iKdPMxVfjetlIMnzkF895um3uTe1WBfNy3SbJ9eaxbOhBqtsq9W1qkM+mss227mN", + "N0pPWzZzy6q29XFABxMqIiemzdzTUbQlVBW762r6Icf7l7pPz/YO9+3rAt/zaN/1pawdOucvvxNV3+Gm", + "OpuGxTfseY95NB+73+b+1z+n72UBHGsexhxgkuVC91Y2ulT3mS+w0l1Gqy/V6x7llIFrHCJwjRiHW2Wi", + "Bkq7w0aXqkBKUZmYRq2mnTyNAWz26G8RdW8A5xV3x4aZ1OJ22CPUs+64ai8v591Lx19WN/u2IevmTtf3", + "U+8+AJ6oPq/t7CbCNdVNZnqU+5ka9Ej73ryjujkb7G8Jnt3Rz6Z10N3Z4ptqkbdJDV+DOzaKju0ufY6w", + "uIRlYFDsa++303Vz/pvVTQU+2FjuzjbNfjjF3rbXXVvuLZCr7lj/3PSdKU0buu8t/X03rf1UOaKr2FrB", + "gK4RAThWXeoBzxdF2MfKXkU/y619kf4AM7EzfPEIudLvoZ0aQeShrzNeR1G1f/f7SqqfMgNstYr6fgnG", + "2Y+eYCyrqwcmGC2T5TmfK3rwFf01h6SDan07+c4oskcvjnCesegu0aY78chX9PDr8Bl1I+nuCdWYXx//", + "TLzNLTt3Mq7O6uzqCkgi043W/MBoLsxdNFy7WHx3qRxcS1ZWkb1eS1q/ItHdTtB/EKH8Wd3Wxd/uErd7", + "c/GGJW9lsdtPlv5ZhLezsuSsxHtgUZLvLRK0YUpikaALwfJQ5OynTD01mRr7O9r6SF5wwGCau78Vtfvp", + "+5rkcYvFN03O/JSQnxIy/z7BUp35dj9Y6hRDf5asTM/8FMWNF/9RBPHhU5RWUrAph3+uWmwtcRuazW6v", + "VcDeOpfyE+A/WOa79enzXb2Pq78vfrfk87CbRdaHDHdQ2ZctzXe9tn5HLzGZaxWaezbjTpr1Ki/9Lfwf", + "TndptHdfddHMr7nUx0fYdbGj9ebza5rvRTSFmKjW8yNJajOBWxeM+rrdRzQc3OLe9LSffs1xeDVRGnii", + "y1InVVewmo4ZuTwzhfZ2obrBYjWJUgsetWwbmqILbDmu+OH28+3/BQAA//8+kDDk07YAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/dm/openapi/gen.types.go b/dm/openapi/gen.types.go index 5a3bd34fcc7..25731a6c3ff 100644 --- a/dm/openapi/gen.types.go +++ b/dm/openapi/gen.types.go @@ -404,14 +404,23 @@ type StartTaskRequest struct { // whether to remove meta database in downstream database RemoveMeta *bool `json:"remove_meta,omitempty"` + // time duration of safe mode + SafeModeTimeDuration *string `json:"safe_mode_time_duration,omitempty"` + // source name list SourceNameList *SourceNameList `json:"source_name_list,omitempty"` + + // task start time + StartTime *string `json:"start_time,omitempty"` } // StopTaskRequest defines model for StopTaskRequest. type StopTaskRequest struct { // source name list SourceNameList *SourceNameList `json:"source_name_list,omitempty"` + + // time duration waiting task stop + TimeoutDuration *string `json:"timeout_duration,omitempty"` } // SubTaskStatus defines model for SubTaskStatus. diff --git a/dm/openapi/spec/dm.yaml b/dm/openapi/spec/dm.yaml index 0f88624b28e..a8a88239cf9 100644 --- a/dm/openapi/spec/dm.yaml +++ b/dm/openapi/spec/dm.yaml @@ -2173,19 +2173,21 @@ components: description: whether to remove meta database in downstream database source_name_list: $ref: "#/components/schemas/SourceNameList" - # start_time: - # type: string - # example: "2006-01-02 15:04:05" - # description: task start time - # safe_mode_time_duration: - # example: "1s" - # description: time duration of safe mode + start_time: + type: string + example: "2006-01-02 15:04:05" + description: task start time + safe_mode_time_duration: + type: string + example: "10s" + description: time duration of safe mode StopTaskRequest: type: object properties: - # timeout_duration: - # example: "15s" - # description: time duration waiting task stop + timeout_duration: + type: string + example: "15s" + description: time duration waiting task stop source_name_list: $ref: "#/components/schemas/SourceNameList" UpdateTaskRequest: diff --git a/dm/pkg/ha/task_cli_args.go b/dm/pkg/ha/task_cli_args.go index 4b9d3483bb7..6f6406bf611 100644 --- a/dm/pkg/ha/task_cli_args.go +++ b/dm/pkg/ha/task_cli_args.go @@ -84,9 +84,15 @@ func DeleteAllTaskCliArgs(cli *clientv3.Client, taskName string) error { } // DeleteTaskCliArgs deleted the command line arguments of this task. -func DeleteTaskCliArgs(cli *clientv3.Client, taskName, source string) error { - key := common.TaskCliArgsKeyAdapter.Encode(taskName, source) - op := clientv3.OpDelete(key) - _, _, err := etcdutil.DoTxnWithRepeatable(cli, etcdutil.ThenOpFunc(op)) +func DeleteTaskCliArgs(cli *clientv3.Client, taskName string, sources []string) error { + if len(sources) == 0 { + return nil + } + ops := []clientv3.Op{} + for _, source := range sources { + key := common.TaskCliArgsKeyAdapter.Encode(taskName, source) + ops = append(ops, clientv3.OpDelete(key)) + } + _, _, err := etcdutil.DoTxnWithRepeatable(cli, etcdutil.ThenOpFunc(ops...)) return err } diff --git a/dm/pkg/ha/task_cli_args_test.go b/dm/pkg/ha/task_cli_args_test.go index 160268e2da5..f1436f3c501 100644 --- a/dm/pkg/ha/task_cli_args_test.go +++ b/dm/pkg/ha/task_cli_args_test.go @@ -62,7 +62,7 @@ func (t *testForEtcd) TestTaskCliArgs(c *C) { c.Assert(*ret, Equals, args) // test delete one source - err = DeleteTaskCliArgs(etcdTestCli, task, source1) + err = DeleteTaskCliArgs(etcdTestCli, task, []string{source1}) c.Assert(err, IsNil) checkNotExist(source1) diff --git a/dm/pkg/terror/error_list.go b/dm/pkg/terror/error_list.go index c47dcf68278..f054d077c68 100644 --- a/dm/pkg/terror/error_list.go +++ b/dm/pkg/terror/error_list.go @@ -915,7 +915,7 @@ var ( ErrConfigNeedUniqueTaskName = New(codeConfigNeedUniqueTaskName, ClassConfig, ScopeInternal, LevelMedium, "must specify a unique task name", "Please check the `name` config in task configuration file.") ErrConfigInvalidTaskMode = New(codeConfigInvalidTaskMode, ClassConfig, ScopeInternal, LevelMedium, "please specify right task-mode, support `full`, `incremental`, `all`", "Please check the `task-mode` config in task configuration file.") ErrConfigNeedTargetDB = New(codeConfigNeedTargetDB, ClassConfig, ScopeInternal, LevelMedium, "must specify target-database", "Please check the `target-database` config in task configuration file.") - ErrConfigMetadataNotSet = New(codeConfigMetadataNotSet, ClassConfig, ScopeInternal, LevelMedium, "mysql-instance(%d) must set meta for task-mode %s", "Please check the `meta` config in task configuration file.") + ErrConfigMetadataNotSet = New(codeConfigMetadataNotSet, ClassConfig, ScopeInternal, LevelMedium, "mysql-instance(%s) must set meta for task-mode %s", "Please check the `meta` config in task configuration file.") ErrConfigRouteRuleNotFound = New(codeConfigRouteRuleNotFound, ClassConfig, ScopeInternal, LevelMedium, "mysql-instance(%d)'s route-rules %s not exist in routes", "Please check the `route-rules` config in task configuration file.") ErrConfigFilterRuleNotFound = New(codeConfigFilterRuleNotFound, ClassConfig, ScopeInternal, LevelMedium, "mysql-instance(%d)'s filter-rules %s not exist in filters", "Please check the `filter-rules` config in task configuration file.") ErrConfigColumnMappingNotFound = New(codeConfigColumnMappingNotFound, ClassConfig, ScopeInternal, LevelMedium, "mysql-instance(%d)'s column-mapping-rules %s not exist in column-mapping", "Please check the `column-mapping-rules` config in task configuration file.") diff --git a/dm/syncer/syncer.go b/dm/syncer/syncer.go index 0db696b65a7..b48638c2fe6 100644 --- a/dm/syncer/syncer.go +++ b/dm/syncer/syncer.go @@ -1470,6 +1470,10 @@ func (s *Syncer) waitBeforeRunExit(ctx context.Context) { s.tctx.L().Info("received subtask's done, try graceful stop") needToExitTime := time.Now() s.waitTransactionLock.Lock() + + failpoint.Inject("checkWaitDuration", func(_ failpoint.Value) { + s.isTransactionEnd = false + }) if s.isTransactionEnd { s.waitXIDJob.Store(int64(waitComplete)) s.waitTransactionLock.Unlock() @@ -1498,6 +1502,17 @@ func (s *Syncer) waitBeforeRunExit(ctx context.Context) { s.runCancel() return } + failpoint.Inject("checkWaitDuration", func(val failpoint.Value) { + if testDuration, testError := time.ParseDuration(val.(string)); testError == nil { + if testDuration.Seconds() == waitDuration.Seconds() { + panic("success check wait_time_on_stop !!!") + } else { + s.tctx.L().Error("checkWaitDuration fail", zap.Duration("testDuration", testDuration), zap.Duration("waitDuration", waitDuration)) + } + } else { + s.tctx.L().Error("checkWaitDuration error", zap.Error(testError)) + } + }) select { case <-s.runCtx.Ctx.Done(): @@ -2196,7 +2211,7 @@ func (s *Syncer) Run(ctx context.Context) (err error) { } // set exitSafeModeTS when meet first binlog - if s.firstMeetBinlogTS == nil && s.cliArgs != nil && s.cliArgs.SafeModeDuration != "" { + if s.firstMeetBinlogTS == nil && s.cliArgs != nil && s.cliArgs.SafeModeDuration != "" && int64(e.Header.Timestamp) != 0 && e.Header.EventType != replication.FORMAT_DESCRIPTION_EVENT { if checkErr := s.initSafeModeExitTS(int64(e.Header.Timestamp)); checkErr != nil { return checkErr } diff --git a/dm/tests/openapi/client/openapi_task_check b/dm/tests/openapi/client/openapi_task_check index 38ea1240482..46f3d2473f7 100755 --- a/dm/tests/openapi/client/openapi_task_check +++ b/dm/tests/openapi/client/openapi_task_check @@ -147,6 +147,66 @@ def create_noshard_task_success(task_name, tartget_table_name=""): print("create_noshard_task_success resp=", resp.json()) assert resp.status_code == 201 +def create_incremental_task_with_gitd_success(task_name,binlog_name1,binlog_pos1,binlog_gtid1,binlog_name2,binlog_pos2,binlog_gtid2): + task = { + "name": task_name, + "task_mode": "incremental", + "meta_schema": "dm_meta", + "enhance_online_schema_change": True, + "on_duplicate": "error", + "target_config": { + "host": "127.0.0.1", + "port": 4000, + "user": "root", + "password": "", + }, + "table_migrate_rule": [ + { + "source": { + "source_name": SOURCE1_NAME, + "schema": "openapi", + "table": "*", + }, + "target": {"schema": "openapi", "table": ""}, + }, + { + "source": { + "source_name": SOURCE2_NAME, + "schema": "openapi", + "table": "*", + }, + "target": {"schema": "openapi", "table": ""}, + }, + ], + "source_config": { + "source_conf": [ + {"source_name": SOURCE1_NAME}, + {"source_name": SOURCE2_NAME}, + ], + }, + } + + if binlog_pos1 != "": + task["source_config"] = { + "source_conf": [ + { + "source_name": SOURCE1_NAME, + "binlog_name": binlog_name1, + "binlog_pos": int(binlog_pos1), + "binlog_gtid": binlog_gtid1, + }, + { + "source_name": SOURCE2_NAME, + "binlog_name": binlog_name2, + "binlog_pos": int(binlog_pos2), + "binlog_gtid": binlog_gtid2, + }, + ], + } + + resp = requests.post(url=API_ENDPOINT, json={"task": task}) + print("create_incremental_task_with_gitd_success resp=", resp.json()) + assert resp.status_code == 201 def create_shard_task_success(): task = { @@ -219,6 +279,43 @@ def start_task_success(task_name, source_name): print("start_task_failed resp=", resp.json()) assert resp.status_code == 200 +def start_task_failed(task_name, source_name, check_result): + url = API_ENDPOINT + "/" + task_name + "/start" + req = {} + if source_name != "": + req = {"source_name_list": [source_name], "remove_meta": True} + resp = requests.post(url=url, json=req) + if resp.status_code == 200: + print("start_task_failed resp should not be 200") + assert resp.status_code == 400 + print("start_task_failed resp=", resp.json()) + assert check_result in resp.json()["error_msg"] + +def start_task_with_condition(task_name, start_time, duration, is_success, check_result): + url = API_ENDPOINT + "/" + task_name + "/start" + req = {"start_time": start_time, "safe_mode_time_duration": duration} + resp = requests.post(url=url, json=req) + if is_success == "success": + if resp.status_code != 200: + print("start_task_with_condition_failed resp=", resp.json()) + assert resp.status_code == 200 + print("start_task_with_condition success") + else: + if resp.status_code == 200: + print("start_task_with_condition_failed resp should not be 200") + assert resp.status_code == 400 + print("start_task_with_condition resp=", resp.json()) + assert check_result in resp.json()["error_msg"] + +def stop_task_with_condition(task_name, source_name, timeout_duration): + url = API_ENDPOINT + "/" + task_name + "/stop" + req = {"timeout_duration": timeout_duration} + if source_name != "": + req = {"source_name_list": [source_name], "timeout_duration": timeout_duration} + resp = requests.post(url=url, json=req) + if resp.status_code != 200: + print("stop_task_failed resp=", resp.json()) + assert resp.status_code == 200 def stop_task_success(task_name, source_name): url = API_ENDPOINT + "/" + task_name + "/stop" @@ -230,7 +327,6 @@ def stop_task_success(task_name, source_name): print("stop_task_failed resp=", resp.json()) assert resp.status_code == 200 - def delete_task_success(task_name): resp = requests.delete(url=API_ENDPOINT + "/" + task_name) assert resp.status_code == 204 @@ -607,10 +703,14 @@ if __name__ == "__main__": "create_task_failed": create_task_failed, "create_noshard_task_success": create_noshard_task_success, "create_shard_task_success": create_shard_task_success, + "create_incremental_task_with_gitd_success": create_incremental_task_with_gitd_success, "delete_task_failed": delete_task_failed, "delete_task_success": delete_task_success, "delete_task_with_force_success": delete_task_with_force_success, "start_task_success": start_task_success, + "start_task_failed": start_task_failed, + "start_task_with_condition": start_task_with_condition, + "stop_task_with_condition": stop_task_with_condition, "stop_task_success": stop_task_success, "get_task_list": get_task_list, "get_task_list_with_status": get_task_list_with_status, diff --git a/dm/tests/openapi/run.sh b/dm/tests/openapi/run.sh index f2e95d1caaf..59c4ad0ac5f 100644 --- a/dm/tests/openapi/run.sh +++ b/dm/tests/openapi/run.sh @@ -610,10 +610,12 @@ function test_delete_task_with_stopped_downstream() { # create source successfully openapi_source_check "create_source1_success" + # create source successfully openapi_source_check "create_source2_success" # get source list success openapi_source_check "list_source_success" 2 + # create no shard task success openapi_task_check "create_noshard_task_success" $task_name $target_table_name run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ @@ -637,6 +639,302 @@ function test_delete_task_with_stopped_downstream() { echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: DELETE TASK WITH STOPPED DOWNSTREAM SUCCESS" } +function test_start_task_with_condition() { + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: START TASK WITH CONDITION" + prepare_database + run_sql_tidb "DROP DATABASE if exists openapi;" + + # create source successfully + openapi_source_check "create_source1_success" + openapi_source_check "list_source_success" 1 + + # get source status success + openapi_source_check "get_source_status_success" "mysql-01" + # create source successfully + openapi_source_check "create_source2_success" + # get source list success + openapi_source_check "list_source_success" 2 + + # get source status success + openapi_source_check "get_source_status_success" "mysql-02" + + # incremental task no source meta and start time, still error + task_name="incremental_task_no_source_meta" + run_sql_source1 "CREATE TABLE openapi.t1(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t2(i TINYINT, j INT UNIQUE KEY);" + + openapi_task_check "create_incremental_task_with_gitd_success" $task_name "" "" "" "" "" "" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Stopped\"" 2 + + check_result="must set meta for task-mode incremental" + openapi_task_check "start_task_failed" $task_name "" "$check_result" + openapi_task_check "delete_task_with_force_success" "$task_name" + openapi_task_check "get_task_list" 0 + + # incremental task use gtid + prepare_database + run_sql_tidb "DROP DATABASE if exists openapi;" + task_name="incremental_task_use_gtid" + run_sql_source1 "CREATE TABLE openapi.t1(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t2(i TINYINT, j INT UNIQUE KEY);" + + master_status1=($(get_master_status $MYSQL_HOST1 $MYSQL_PORT1)) + master_status2=($(get_master_status $MYSQL_HOST2 $MYSQL_PORT2)) + openapi_task_check "create_incremental_task_with_gitd_success" $task_name ${master_status1[0]} ${master_status1[1]} ${master_status1[2]} ${master_status2[0]} ${master_status2[1]} ${master_status2[2]} + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Stopped\"" 2 + openapi_task_check "start_task_success" $task_name "" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Running\"" 2 + + run_sql_tidb 'CREATE DATABASE openapi;' + run_sql_source1 "CREATE TABLE openapi.t3(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t4(i TINYINT, j INT UNIQUE KEY);" + + run_sql_tidb_with_retry "show tables in openapi;" "t3" + run_sql_tidb_with_retry "show tables in openapi;" "t4" + run_sql_tidb_with_retry "SELECT count(1) FROM information_schema.tables WHERE table_schema = 'openapi';" "count(1): 2" + + openapi_task_check "stop_task_success" "$task_name" "" + openapi_task_check "delete_task_with_force_success" "$task_name" + openapi_task_check "get_task_list" 0 + + # incremental task use start_time + prepare_database + run_sql_tidb "DROP DATABASE if exists openapi;" + task_name="incremental_task_use_start_time" + run_sql_source1 "CREATE TABLE openapi.t1(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t2(i TINYINT, j INT UNIQUE KEY);" + + openapi_task_check "create_incremental_task_with_gitd_success" $task_name "" "" "" "" "" "" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Stopped\"" 2 + sleep 2 + start_time=$(date '+%Y-%m-%d %T') + sleep 2 + duration="" + is_success="success" + check_result="" + run_sql_tidb 'CREATE DATABASE openapi;' + run_sql_source1 "CREATE TABLE openapi.t3(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t4(i TINYINT, j INT UNIQUE KEY);" + openapi_task_check "start_task_with_condition" $task_name "$start_time" "$duration" "$is_success" "$check_result" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Running\"" 2 + + run_sql_tidb_with_retry "show tables in openapi;" "t3" + run_sql_tidb_with_retry "show tables in openapi;" "t4" + run_sql_tidb_with_retry "SELECT count(1) FROM information_schema.tables WHERE table_schema = 'openapi';" "count(1): 2" + + openapi_task_check "stop_task_success" "$task_name" "" + openapi_task_check "delete_task_with_force_success" "$task_name" + openapi_task_check "get_task_list" 0 + + # incremental task use start_time, but time is after create table + prepare_database + run_sql_tidb "DROP DATABASE if exists openapi;" + task_name="incremental_task_use_start_time_after_create" + run_sql_source1 "CREATE TABLE openapi.t1(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t2(i TINYINT, j INT UNIQUE KEY);" + + openapi_task_check "create_incremental_task_with_gitd_success" $task_name "" "" "" "" "" "" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Stopped\"" 2 + sleep 2 + start_time=$(date '+%Y-%m-%d %T') + sleep 2 + duration="" + is_success="success" + check_result="" + run_sql_tidb 'CREATE DATABASE openapi;' + run_sql_source1 "INSERT INTO openapi.t1(i,j) VALUES (1, 2);" + run_sql_source2 "INSERT INTO openapi.t2(i,j) VALUES (3, 4);" + openapi_task_check "start_task_with_condition" $task_name "$start_time" "$duration" "$is_success" "$check_result" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "Table 'openapi.*' doesn't exist" 2 + + openapi_task_check "stop_task_success" "$task_name" "" + openapi_task_check "delete_task_with_force_success" "$task_name" + openapi_task_check "get_task_list" 0 + + # incremental task both gtid and start_time, start_time first + prepare_database + run_sql_tidb "DROP DATABASE if exists openapi;" + task_name="incremental_task_both_gtid_start_time" + run_sql_source1 "CREATE TABLE openapi.t1(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t2(i TINYINT, j INT UNIQUE KEY);" + master_status1=($(get_master_status $MYSQL_HOST1 $MYSQL_PORT1)) + master_status2=($(get_master_status $MYSQL_HOST2 $MYSQL_PORT2)) + openapi_task_check "create_incremental_task_with_gitd_success" $task_name ${master_status1[0]} ${master_status1[1]} ${master_status1[2]} ${master_status2[0]} ${master_status2[1]} ${master_status2[2]} + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Stopped\"" 2 + run_sql_source1 "CREATE TABLE openapi.t3(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t4(i TINYINT, j INT UNIQUE KEY);" + sleep 2 + start_time=$(date '+%Y-%m-%d %T') + sleep 2 + duration="" + is_success="success" + check_result="" + run_sql_tidb 'CREATE DATABASE openapi;' + run_sql_source1 "CREATE TABLE openapi.t5(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t6(i TINYINT, j INT UNIQUE KEY);" + openapi_task_check "start_task_with_condition" $task_name "$start_time" "$duration" "$is_success" "$check_result" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Running\"" 2 + + run_sql_tidb_with_retry "show tables in openapi;" "t5" + run_sql_tidb_with_retry "show tables in openapi;" "t6" + run_sql_tidb_with_retry "SELECT count(1) FROM information_schema.tables WHERE table_schema = 'openapi';" "count(1): 2" + + openapi_task_check "stop_task_success" "$task_name" "" + openapi_task_check "delete_task_with_force_success" "$task_name" + openapi_task_check "get_task_list" 0 + + # incremental task no duration has error + export GO_FAILPOINTS='github.com/pingcap/tiflow/dm/syncer/SafeModeInitPhaseSeconds=return(0)' + kill_dm_worker + check_port_offline $WORKER1_PORT 20 + check_port_offline $WORKER2_PORT 20 + + # run dm-worker1 + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + # run dm-worker2 + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + openapi_source_check "list_source_success" 2 + + prepare_database + run_sql_tidb "DROP DATABASE if exists openapi;" + task_name="incremental_task_no_duration_but_error" + run_sql_source1 "CREATE TABLE openapi.t1(i TINYINT, j INT UNIQUE KEY);" + run_sql_source2 "CREATE TABLE openapi.t2(i TINYINT, j INT UNIQUE KEY);" + + sleep 2 + start_time=$(date '+%Y-%m-%d %T') + sleep 2 + duration="" + is_success="success" + check_result="" + + run_sql_source1 "INSERT INTO openapi.t1(i,j) VALUES (1, 2);" + run_sql_source2 "INSERT INTO openapi.t2(i,j) VALUES (1, 2);" + run_sql_source1 "INSERT INTO openapi.t1(i,j) VALUES (3, 4);" + run_sql_source2 "INSERT INTO openapi.t2(i,j) VALUES (3, 4);" + # mock already sync data to downstream + run_sql_tidb 'CREATE DATABASE openapi;' + run_sql_tidb "CREATE TABLE openapi.t1(i TINYINT, j INT UNIQUE KEY);" + run_sql_tidb "CREATE TABLE openapi.t2(i TINYINT, j INT UNIQUE KEY);" + run_sql_tidb "INSERT INTO openapi.t1(i,j) VALUES (1, 2);" + run_sql_tidb "INSERT INTO openapi.t2(i,j) VALUES (1, 2);" + + openapi_task_check "create_incremental_task_with_gitd_success" $task_name "" "" "" "" "" "" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Stopped\"" 2 + openapi_task_check "start_task_with_condition" $task_name "$start_time" "$duration" "$is_success" "$check_result" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "Duplicate entry" 2 + + # set duration and start again + openapi_task_check "stop_task_success" "$task_name" "" + duration="600s" + openapi_task_check "start_task_with_condition" $task_name "$start_time" "$duration" "$is_success" "$check_result" + + run_sql_tidb_with_retry "SELECT count(1) FROM openapi.t1;" "count(1): 2" + run_sql_tidb_with_retry "SELECT count(1) FROM openapi.t2;" "count(1): 2" + + openapi_task_check "stop_task_success" "$task_name" "" + openapi_task_check "delete_task_with_force_success" "$task_name" + openapi_task_check "get_task_list" 0 + + clean_cluster_sources_and_tasks + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: START TASK WITH CONDITION SUCCESS" +} + +function test_stop_task_with_condition() { + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: STOP TASK WITH CONDITION" + prepare_database + run_sql_tidb "DROP DATABASE if exists openapi;" + + # create source successfully + openapi_source_check "create_source1_success" + openapi_source_check "list_source_success" 1 + + # get source status success + openapi_source_check "get_source_status_success" "mysql-01" + # create source successfully + openapi_source_check "create_source2_success" + # get source list success + openapi_source_check "list_source_success" 2 + + # get source status success + openapi_source_check "get_source_status_success" "mysql-02" + + # test wait_time_on_stop + export GO_FAILPOINTS='github.com/pingcap/tiflow/dm/syncer/recordAndIgnorePrepareTime=return();github.com/pingcap/tiflow/dm/syncer/checkWaitDuration=return("200s")' + kill_dm_worker + check_port_offline $WORKER1_PORT 20 + check_port_offline $WORKER2_PORT 20 + + # run dm-worker1 + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + # run dm-worker2 + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + + task_name="test_wait_time_on_stop" + # create no shard task success + openapi_task_check "create_noshard_task_success" $task_name "" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Stopped\"" 2 + + timeout_duration="200s" + + openapi_task_check "start_task_success" $task_name "" + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + "query-status $task_name" \ + "\"stage\": \"Running\"" 2 + init_noshard_data + check_sync_diff $WORK_DIR $cur/conf/diff_config_no_shard.toml + openapi_task_check "stop_task_with_condition" "$task_name" "" "$timeout_duration" + echo "error check" + check_log_contain_with_retry 'panic: success check wait_time_on_stop !!!' $WORK_DIR/worker1/log/stdout.log + check_log_contain_with_retry 'panic: success check wait_time_on_stop !!!' $WORK_DIR/worker2/log/stdout.log + + # clean + export GO_FAILPOINTS='' + kill_dm_worker + check_port_offline $WORKER1_PORT 20 + check_port_offline $WORKER2_PORT 20 + + # run dm-worker1 + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + # run dm-worker2 + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + openapi_task_check "delete_task_with_force_success" "$task_name" + openapi_task_check "get_task_list" 0 + + clean_cluster_sources_and_tasks + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: START TASK WITH CONDITION SUCCESS" +} + function test_cluster() { # list master and worker node openapi_cluster_check "list_master_success" 2 @@ -682,6 +980,8 @@ function run() { test_complex_operations_of_source_and_task test_task_with_ignore_check_items test_delete_task_with_stopped_downstream + test_start_task_with_condition + test_stop_task_with_condition # NOTE: this test case MUST running at last, because it will offline some members of cluster test_cluster