User friendly update

README.md

@@ -5,6 +5,7 @@ Milvus-backup is a tool to backup and restore Milvus data. It can be used as a c
 Milvus-backup needs to visit Milvus proxy and minio cluster. Related config can be edited in `configs/backup.yaml`. 
 > **Note**
 > Please make sure the config of Minio is correct. There are some differences for the default value of Minio config when the Milvus is deployed by docker-compose and k8s.
+> Remind it is not supported to backup data to a local path. Backup data is also stored in minio or some other kind of object storage your milvus used.
 
 |field|docker-compose |k8s|
 |---|---|---|

configs/backup.yaml

@@ -26,12 +26,13 @@ minio:
   accessKeyID: minioadmin # accessKeyID of MinIO/S3
   secretAccessKey: minioadmin # MinIO/S3 encryption string
   useSSL: false # Access to MinIO/S3 with SSL
-  bucketName: "a-bucket" # Bucket name in MinIO/S3
-  rootPath: files # The root path where the message is stored in MinIO/S3
   useIAM: false
   cloudProvider: "aws"
   iamEndpoint: ""
+
+  bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance
+  rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance
 
-  backupBucketName: "a-bucket"
-  backupRootPath: "backup"
+  backupBucketName: "a-bucket" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath
+  backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath
 

core/backup_context.go

@@ -169,17 +169,15 @@ func (b BackupContext) CreateBackup(ctx context.Context, request *backuppb.Creat
 
 	// backup name validate
 	if request.GetBackupName() != "" {
-		getResp := b.GetBackup(b.ctx, &backuppb.GetBackupRequest{
-			BackupName: request.GetBackupName(),
-		})
-		if getResp.GetCode() == backuppb.ResponseCode_Request_Object_Not_Found {
-			log.Info("backup not exist", zap.String("backup_name", request.GetBackupName()))
-		} else if getResp.GetCode() != backuppb.ResponseCode_Success {
-			log.Error("fail in GetBackup", zap.String("msg", getResp.GetMsg()))
+		exist, err := b.storageClient.Exist(b.ctx, b.backupBucketName, b.backupRootPath+SEPERATOR+request.GetBackupName())
+		if err != nil {
+			errMsg := fmt.Sprintf("fail to check whether exist backup with name: %s", request.GetBackupName())
+			log.Error(errMsg, zap.Error(err))
 			resp.Code = backuppb.ResponseCode_Fail
-			resp.Msg = getResp.GetMsg()
+			resp.Msg = errMsg + "/n" + err.Error()
 			return resp
-		} else if getResp.GetData() != nil {
+		}
+		if exist {
 			errMsg := fmt.Sprintf("backup already exist with the name: %s", request.GetBackupName())
 			log.Error(errMsg)
 			resp.Code = backuppb.ResponseCode_Parameter_Error
@@ -326,22 +324,13 @@ func (b BackupContext) executeCreateBackup(ctx context.Context, request *backupp
 		}
 
 		hasIndex := false
-		var indexInfo *backuppb.IndexInfo
+		indexInfos := make([]*backuppb.IndexInfo, 0)
 		log.Info("try to get index",
 			zap.String("collection_name", completeCollection.Name))
 		for _, field := range completeCollection.Schema.Fields {
 			if field.DataType != entity.FieldTypeBinaryVector && field.DataType != entity.FieldTypeFloatVector {
 				continue
 			}
-
-			//indexState, err := b.milvusClient.GetIndexState(b.ctx, completeCollection.Name, field.Name)
-			//if err != nil {
-			//	log.Error("fail in GetIndexState", zap.Error(err))
-			//	return backupInfo, err
-			//}
-			//if indexState == 0 {
-			//	continue
-			//}
 			fieldIndex, err := b.milvusClient.DescribeIndex(b.ctx, completeCollection.Name, field.Name)
 			if err != nil {
 				if strings.HasPrefix(err.Error(), "index doesn't exist") {
@@ -361,11 +350,13 @@ func (b BackupContext) executeCreateBackup(ctx context.Context, request *backupp
 				zap.String("field_name", field.Name),
 				zap.Any("index info", fieldIndex))
 			if len(fieldIndex) != 0 {
-				indexInfo = &backuppb.IndexInfo{
-					Name:      fieldIndex[0].Name(),
+				indexInfo := &backuppb.IndexInfo{
+					FieldName: field.Name,
+					IndexName: fieldIndex[0].Name(),
 					IndexType: string(fieldIndex[0].IndexType()),
 					Params:    fieldIndex[0].Params(),
 				}
+				indexInfos = append(indexInfos, indexInfo)
 				hasIndex = true
 			}
 		}
@@ -382,7 +373,7 @@ func (b BackupContext) executeCreateBackup(ctx context.Context, request *backupp
 			ShardsNum:        completeCollection.ShardNum,
 			ConsistencyLevel: backuppb.ConsistencyLevel(completeCollection.ConsistencyLevel),
 			HasIndex:         hasIndex,
-			IndexInfo:        indexInfo,
+			IndexInfos:       indexInfos,
 		}
 		collectionBackupInfos = append(collectionBackupInfos, collectionBackup)
 	}
@@ -1287,7 +1278,7 @@ func (b BackupContext) readBackup(ctx context.Context, bucketName string, backup
 		return nil, err
 	}
 	if !exist {
-		log.Warn("read backup meta file not exist, you may need to create it first", zap.String("path", backupMetaPath), zap.Error(err))
+		log.Warn("read backup meta file not exist", zap.String("path", backupMetaPath), zap.Error(err))
 		return nil, err
 	}
 

core/backup_meta.go

@@ -58,7 +58,7 @@ func treeToLevel(backup *backuppb.BackupInfo) (LeveledBackupInfo, error) {
 			BackupTimestamp:  collectionBack.GetBackupTimestamp(),
 			Size:             collectionBack.GetSize(),
 			HasIndex:         collectionBack.GetHasIndex(),
-			IndexInfo:        collectionBack.GetIndexInfo(),
+			IndexInfos:       collectionBack.GetIndexInfos(),
 			LoadState:        collectionBack.GetLoadState(),
 		}
 		collections = append(collections, cloneCollectionBackup)
@@ -260,7 +260,7 @@ func SimpleBackupResponse(input *backuppb.BackupInfoResponse) *backuppb.BackupIn
 			CollectionName:  coll.GetCollectionName(),
 			BackupTimestamp: coll.GetBackupTimestamp(),
 			HasIndex:        coll.GetHasIndex(),
-			IndexInfo:       coll.GetIndexInfo(),
+			IndexInfos:      coll.GetIndexInfos(),
 			LoadState:       coll.GetLoadState(),
 			Schema:          coll.GetSchema(),
 			Size:            coll.GetSize(),

core/backup_server.go

@@ -66,6 +66,7 @@ func (s *Server) registerHTTPServer() {
 	}
 	ginHandler := gin.Default()
 	apiv1 := ginHandler.Group(API_V1_PREFIX)
+	ginHandler.Any("", wrapHandler(handleHello))
 	NewHandlers(s.backupContext).RegisterRoutesTo(apiv1)
 	http.Handle("/", ginHandler)
 	s.engine = ginHandler
@@ -79,6 +80,11 @@ func (s *Server) registerProfilePort() {
 	}()
 }
 
+func handleHello(c *gin.Context) (interface{}, error) {
+	c.String(200, "Hello, This is backup service")
+	return nil, nil
+}
+
 type Handlers struct {
 	backupContext *BackupContext
 }
@@ -92,7 +98,7 @@ func NewHandlers(backupContext *BackupContext) *Handlers {
 
 // RegisterRouters registers routes to given router
 func (h *Handlers) RegisterRoutesTo(router gin.IRouter) {
-	router.GET(HELLO_API, wrapHandler(h.handleHello))
+	router.GET(HELLO_API, wrapHandler(handleHello))
 	router.POST(CREATE_BACKUP_API, wrapHandler(h.handleCreateBackup))
 	router.GET(LIST_BACKUPS_API, wrapHandler(h.handleListBackups))
 	router.GET(GET_BACKUP_API, wrapHandler(h.handleGetBackup))
@@ -112,11 +118,6 @@ func wrapHandler(handle handlerFunc) gin.HandlerFunc {
 	}
 }
 
-func (h *Handlers) handleHello(c *gin.Context) (interface{}, error) {
-	c.String(200, "Hello, This is backup service")
-	return nil, nil
-}
-
 // CreateBackup Create backup interface
 // @Summary Create backup interface
 // @Description Create a backup with the given name and collections

core/proto/backup.proto

@@ -28,9 +28,10 @@ enum ResponseCode {
 }
 
 message IndexInfo {
-  string name = 1;
-  string index_type = 2;
-  map<string, string> params = 3;
+  string field_name = 1;
+  string index_name = 2;
+  string index_type = 3;
+  map<string, string> params = 4;
 }
 
 /**
@@ -53,7 +54,7 @@ message CollectionBackupInfo {
   uint64 backup_timestamp = 14;
   int64 size = 15;
   bool has_index = 16;
-  IndexInfo index_info = 17;
+  repeated IndexInfo index_infos = 17;
   string load_state = 18;
 }