Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support return simple response #26

Merged
merged 3 commits into from
Nov 28, 2022
Merged

Support return simple response #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
28de60c
Support return simple response
wayblink Nov 28, 2022
504ff03
Rename response code
wayblink Nov 28, 2022
7d62e8e
fix create backup bug
wayblink Nov 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions configs/backup.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ log:
file:
rootPath: "logs/backup.log"

http:
simpleResponse: true

# milvus proxy address, compatible to milvus.yaml
milvus:
address: localhost
Expand Down Expand Up @@ -32,5 +35,3 @@ minio:
backupBucketName: "a-bucket"
backupRootPath: "backup"

backup:
storageType: minio # backup data storage engine: minio/S3/GCP
36 changes: 18 additions & 18 deletions core/backup_context.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func (b BackupContext) CreateBackup(ctx context.Context, request *backuppb.Creat
if !b.started {
err := b.Start()
if err != nil {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
return resp
}
Expand All @@ -173,7 +173,7 @@ func (b BackupContext) CreateBackup(ctx context.Context, request *backuppb.Creat
})
if getResp.GetCode() != backuppb.ResponseCode_Success {
log.Error("fail in GetBackup", zap.String("msg", getResp.GetMsg()))
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = getResp.GetMsg()
return resp
}
Expand Down Expand Up @@ -223,7 +223,7 @@ func (b BackupContext) CreateBackup(ctx context.Context, request *backuppb.Creat
task, err := b.executeCreateBackup(ctx, request, backup)
resp.Data = task
if err != nil {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
} else {
resp.Code = backuppb.ResponseCode_Success
Expand Down Expand Up @@ -273,7 +273,7 @@ func (b BackupContext) executeCreateBackup(ctx context.Context, request *backupp
log.Debug(fmt.Sprintf("List %v collections", len(collections)))
toBackupCollections = collections
} else {
toBackupCollections := make([]*entity.Collection, 0)
//toBackupCollections := make([]*entity.Collection, 0)
for _, collectionName := range request.GetCollectionNames() {
exist, err := b.milvusClient.HasCollection(b.ctx, collectionName)
if err != nil {
Expand Down Expand Up @@ -560,7 +560,7 @@ func (b BackupContext) GetBackup(ctx context.Context, request *backuppb.GetBacku
if !b.started {
err := b.Start()
if err != nil {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
return resp
}
Expand Down Expand Up @@ -591,7 +591,7 @@ func (b BackupContext) GetBackup(ctx context.Context, request *backuppb.GetBacku
backup, err := b.readBackup(ctx, request.GetBackupName())
if err != nil {
log.Warn("Fail to read backup", zap.String("backupName", request.GetBackupName()), zap.Error(err))
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
return resp
}
Expand Down Expand Up @@ -619,7 +619,7 @@ func (b BackupContext) ListBackups(ctx context.Context, request *backuppb.ListBa
if !b.started {
err := b.Start()
if err != nil {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
return resp
}
Expand All @@ -629,7 +629,7 @@ func (b BackupContext) ListBackups(ctx context.Context, request *backuppb.ListBa
backupPaths, _, err := b.storageClient.ListWithPrefix(ctx, b.backupBucketName, b.backupRootPath+SEPERATOR, false)
if err != nil {
log.Error("Fail to list backup directory", zap.Error(err))
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
return resp
}
Expand All @@ -645,7 +645,7 @@ func (b BackupContext) ListBackups(ctx context.Context, request *backuppb.ListBa
log.Warn("Fail to read backup",
zap.String("path", backupPath),
zap.String("error", backupResp.GetMsg()))
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = backupResp.Msg
return resp
}
Expand Down Expand Up @@ -694,7 +694,7 @@ func (b BackupContext) DeleteBackup(ctx context.Context, request *backuppb.Delet
if !b.started {
err := b.Start()
if err != nil {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
return resp
}
Expand Down Expand Up @@ -740,7 +740,7 @@ func (b BackupContext) RestoreBackup(ctx context.Context, request *backuppb.Rest
if !b.started {
err := b.Start()
if err != nil {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
return resp
}
Expand All @@ -764,7 +764,7 @@ func (b BackupContext) RestoreBackup(ctx context.Context, request *backuppb.Rest
log.Error("fail to get backup",
zap.String("backupName", request.GetBackupName()),
zap.String("msg", getResp.GetMsg()))
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = getResp.GetMsg()
return resp
}
Expand Down Expand Up @@ -820,14 +820,14 @@ func (b BackupContext) RestoreBackup(ctx context.Context, request *backuppb.Rest
if err != nil {
errorMsg := fmt.Sprintf("fail to check whether the collection is exist, collection_name: %s, err: %s", targetCollectionName, err)
log.Error(errorMsg)
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = errorMsg
return resp
}
if exist {
errorMsg := fmt.Sprintf("The collection to restore already exists, backupCollectName: %s, targetCollectionName: %s", backupCollectionName, targetCollectionName)
log.Error(errorMsg)
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = errorMsg
return resp
}
Expand Down Expand Up @@ -858,7 +858,7 @@ func (b BackupContext) RestoreBackup(ctx context.Context, request *backuppb.Rest
endTask, err := b.executeRestoreBackupTask(ctx, backup, task)
resp.Data = endTask
if err != nil {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
} else {
resp.Code = backuppb.ResponseCode_Success
Expand Down Expand Up @@ -1192,14 +1192,14 @@ func (b *BackupContext) GetRestore(ctx context.Context, request *backuppb.GetRes
if !b.started {
err := b.Start()
if err != nil {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = err.Error()
return resp
}
}

if request.GetId() == "" {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = "empty restore id"
return resp
}
Expand All @@ -1210,7 +1210,7 @@ func (b *BackupContext) GetRestore(ctx context.Context, request *backuppb.GetRes
resp.Data = value
return resp
} else {
resp.Code = backuppb.ResponseCode_Bad_Request
resp.Code = backuppb.ResponseCode_Fail
resp.Msg = "restore id not exist in context"
return resp
}
Expand Down
13 changes: 7 additions & 6 deletions core/backup_context_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ func TestCreateBackup(t *testing.T) {
backup := CreateBackupContext(context, params)

req := &backuppb.CreateBackupRequest{
BackupName: "test_21",
BackupName: "test_21",
CollectionNames: []string{"hello_milvus", "hello_milvus2"},
}
backup.CreateBackup(context, req)
}
Expand Down Expand Up @@ -94,7 +95,7 @@ func TestCreateBackupWithUnexistCollection(t *testing.T) {
CollectionNames: []string{"not_exist"},
}
resp := backup.CreateBackup(context, req)
assert.Equal(t, backuppb.ResponseCode_Bad_Request, resp.GetCode())
assert.Equal(t, backuppb.ResponseCode_Fail, resp.GetCode())
assert.Equal(t, "request backup collection does not exist: not_exist", resp.GetMsg())

// clean
Expand All @@ -121,7 +122,7 @@ func TestCreateBackupWithDuplicateName(t *testing.T) {
BackupName: randBackupName,
}
resp2 := backup.CreateBackup(context, req2)
assert.Equal(t, backuppb.ResponseCode_Bad_Request, resp2.GetCode())
assert.Equal(t, backuppb.ResponseCode_Fail, resp2.GetCode())
assert.Equal(t, fmt.Sprintf("backup already exist with the name: %s", req2.GetBackupName()), resp2.GetMsg())

// clean
Expand All @@ -142,7 +143,7 @@ func TestCreateBackupWithIllegalName(t *testing.T) {
BackupName: randBackupName,
}
resp := backup.CreateBackup(context, req)
assert.Equal(t, backuppb.ResponseCode_Bad_Request, resp.GetCode())
assert.Equal(t, backuppb.ResponseCode_Fail, resp.GetCode())

// clean
backup.DeleteBackup(context, &backuppb.DeleteBackupRequest{
Expand Down Expand Up @@ -199,7 +200,7 @@ func TestGetBackupFaultBackup(t *testing.T) {
backup := backupContext.GetBackup(context, &backuppb.GetBackupRequest{
BackupName: randBackupName,
})
assert.Equal(t, backuppb.ResponseCode_Bad_Request, backup.GetCode())
assert.Equal(t, backuppb.ResponseCode_Fail, backup.GetCode())

// clean
backupContext.DeleteBackup(context, &backuppb.DeleteBackupRequest{
Expand All @@ -217,7 +218,7 @@ func TestGetBackupUnexistBackupName(t *testing.T) {
backup := backupContext.GetBackup(context, &backuppb.GetBackupRequest{
BackupName: "un_exist",
})
assert.Equal(t, backuppb.ResponseCode_Bad_Request, backup.GetCode())
assert.Equal(t, backuppb.ResponseCode_Fail, backup.GetCode())
}

func TestRestoreBackup(t *testing.T) {
Expand Down
79 changes: 79 additions & 0 deletions core/backup_meta.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,3 +211,82 @@ func SegmentMetaPath(backupRootPath, backupName string) string {
func BackupBinlogDirPath(backupRootPath, backupName string) string {
return backupRootPath + SEPERATOR + backupName + SEPERATOR + BINGLOG_DIR
}

func SimpleListBackupsResponse(input *backuppb.ListBackupsResponse) *backuppb.ListBackupsResponse {
simpleBackupInfos := make([]*backuppb.BackupInfo, 0)
for _, backup := range input.GetData() {
simpleBackupInfos = append(simpleBackupInfos, &backuppb.BackupInfo{
Id: backup.GetId(),
Name: backup.GetName(),
StateCode: backup.GetStateCode(),
ErrorMessage: backup.GetErrorMessage(),
BackupTimestamp: backup.GetBackupTimestamp(),
})
}
return &backuppb.ListBackupsResponse{
RequestId: input.GetRequestId(),
Code: input.GetCode(),
Msg: input.GetMsg(),
Data: simpleBackupInfos,
}
}

func SimpleBackupResponse(input *backuppb.BackupInfoResponse) *backuppb.BackupInfoResponse {
backup := input.GetData()

collections := make([]*backuppb.CollectionBackupInfo, 0)
for _, coll := range backup.GetCollectionBackups() {
collections = append(collections, &backuppb.CollectionBackupInfo{
StateCode: coll.GetStateCode(),
ErrorMessage: coll.GetErrorMessage(),
CollectionName: coll.GetCollectionName(),
BackupTimestamp: coll.GetBackupTimestamp(),
})
}
simpleBackupInfo := &backuppb.BackupInfo{
Id: backup.GetId(),
Name: backup.GetName(),
StateCode: backup.GetStateCode(),
ErrorMessage: backup.GetErrorMessage(),
BackupTimestamp: backup.GetBackupTimestamp(),
CollectionBackups: collections,
}
return &backuppb.BackupInfoResponse{
RequestId: input.GetRequestId(),
Code: input.GetCode(),
Msg: input.GetMsg(),
Data: simpleBackupInfo,
}
}

func SimpleRestoreResponse(input *backuppb.RestoreBackupResponse) *backuppb.RestoreBackupResponse {
restore := input.GetData()

collectionRestores := make([]*backuppb.RestoreCollectionTask, 0)
for _, coll := range restore.GetCollectionRestoreTasks() {
collectionRestores = append(collectionRestores, &backuppb.RestoreCollectionTask{
StateCode: coll.GetStateCode(),
ErrorMessage: coll.GetErrorMessage(),
StartTime: coll.GetStartTime(),
EndTime: coll.GetEndTime(),
Progress: coll.GetProgress(),
TargetCollectionName: coll.GetTargetCollectionName(),
})
}

simpleRestore := &backuppb.RestoreBackupTask{
Id: restore.GetId(),
StateCode: restore.GetStateCode(),
ErrorMessage: restore.GetErrorMessage(),
StartTime: restore.GetStartTime(),
EndTime: restore.GetEndTime(),
Progress: restore.GetProgress(),
}

return &backuppb.RestoreBackupResponse{
RequestId: input.GetRequestId(),
Code: input.GetCode(),
Msg: input.GetMsg(),
Data: simpleRestore,
}
}
19 changes: 18 additions & 1 deletion core/backup_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ const (
GET_RESTORE_API = "/get_restore"

API_V1_PREFIX = "/api/v1"

DOCS_API = "/docs/*any"
)

// Server is the Backup Server
Expand Down Expand Up @@ -97,7 +99,7 @@ func (h *Handlers) RegisterRoutesTo(router gin.IRouter) {
router.DELETE(DELETE_BACKUP_API, wrapHandler(h.handleDeleteBackup))
router.POST(RESTORE_BACKUP_API, wrapHandler(h.handleRestoreBackup))
router.GET(GET_RESTORE_API, wrapHandler(h.handleGetRestore))
router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler))
router.GET(DOCS_API, ginSwagger.WrapHandler(swaggerFiles.Handler))
}

// handlerFunc handles http request with gin context
Expand Down Expand Up @@ -128,6 +130,9 @@ func (h *Handlers) handleCreateBackup(c *gin.Context) (interface{}, error) {
json := backuppb.CreateBackupRequest{}
c.BindJSON(&json)
resp := h.backupContext.CreateBackup(h.backupContext.ctx, &json)
if h.backupContext.params.HTTPCfg.SimpleResponse {
resp = SimpleBackupResponse(resp)
}
c.JSON(http.StatusOK, resp)
return nil, nil
}
Expand All @@ -146,6 +151,9 @@ func (h *Handlers) handleListBackups(c *gin.Context) (interface{}, error) {
CollectionName: c.Query("collection_name"),
}
resp := h.backupContext.ListBackups(h.backupContext.ctx, &req)
if h.backupContext.params.HTTPCfg.SimpleResponse {
resp = SimpleListBackupsResponse(resp)
}
c.JSON(http.StatusOK, resp)
return nil, nil
}
Expand All @@ -165,6 +173,9 @@ func (h *Handlers) handleGetBackup(c *gin.Context) (interface{}, error) {
BackupId: c.Query("backup_id"),
}
resp := h.backupContext.GetBackup(h.backupContext.ctx, &req)
if h.backupContext.params.HTTPCfg.SimpleResponse {
resp = SimpleBackupResponse(resp)
}
c.JSON(http.StatusOK, resp)
return nil, nil
}
Expand Down Expand Up @@ -200,6 +211,9 @@ func (h *Handlers) handleRestoreBackup(c *gin.Context) (interface{}, error) {
json := backuppb.RestoreBackupRequest{}
c.BindJSON(&json)
resp := h.backupContext.RestoreBackup(h.backupContext.ctx, &json)
if h.backupContext.params.HTTPCfg.SimpleResponse {
resp = SimpleRestoreResponse(resp)
}
c.JSON(http.StatusOK, resp)
return nil, nil
}
Expand All @@ -219,6 +233,9 @@ func (h *Handlers) handleGetRestore(c *gin.Context) (interface{}, error) {
Id: id,
}
resp := h.backupContext.GetRestore(h.backupContext.ctx, &req)
if h.backupContext.params.HTTPCfg.SimpleResponse {
resp = SimpleRestoreResponse(resp)
}
c.JSON(http.StatusOK, resp)
return nil, nil
}
Loading