Merge pull request #7 from zeripath/lunny/storage_config

Add IterateObjects
zeripath · Sep 28, 2020 · 38d1cc5 · 38d1cc5
2 parents b3af353 + a28ffda
commit 38d1cc5
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 63 deletions.
diff --git a/cmd/dump.go b/cmd/dump.go
@@ -238,39 +238,21 @@ func runDump(ctx *cli.Context) error {
 			fatal("Failed to include repositories: %v", err)
 		}
 
-		switch setting.LFS.Storage.Type {
-		case setting.LocalStorageType:
-			if _, err := os.Stat(setting.LFS.Path); !os.IsNotExist(err) {
-				log.Info("Dumping lfs... %s", setting.LFS.Path)
-				if err := addRecursive(w, "lfs", setting.LFS.Path, verbose); err != nil {
-					fatal("Failed to include lfs: %v", err)
-				}
+		if err := storage.LFS.IterateObjects(func(objPath string, object storage.Object) error {
+			info, err := object.Stat()
+			if err != nil {
+				return err
 			}
-		case setting.MinioStorageType:
-			if err := models.IterateLFS(func(mo *models.LFSMetaObject) error {
-				f, err := storage.LFS.Open(mo.RelativePath())
-				if err != nil {
-					return err
-				}
-				defer f.Close()
-
-				info, err := f.Stat()
-				if err != nil {
-					return err
-				}
 
-				return w.Write(archiver.File{
-					FileInfo: archiver.FileInfo{
-						FileInfo:   info,
-						CustomName: path.Join("data", "lfs", mo.RelativePath()),
-					},
-					ReadCloser: f,
-				})
-			}); err != nil {
-				fatal("Dump LFS failed: %v", err)
-			}
-		default:
-			fatal("Unknown LFS storage type: %s", setting.LFS.Storage.Type)
+			return w.Write(archiver.File{
+				FileInfo: archiver.FileInfo{
+					FileInfo:   info,
+					CustomName: path.Join("data", "lfs", objPath),
+				},
+				ReadCloser: object,
+			})
+		}); err != nil {
+			fatal("Failed to dump LFS objects: %v", err)
 		}
 	}
 
@@ -345,40 +327,21 @@ func runDump(ctx *cli.Context) error {
 		}
 	}
 
-	// attachment default should be under data directory but not always there.
-	switch setting.Attachment.Storage.Type {
-	case setting.LocalStorageType:
-		if _, err := os.Stat(setting.Attachment.Path); !os.IsNotExist(err) {
-			log.Info("Dumping attachment... %s", setting.Attachment.Path)
-			if err := addRecursive(w, "attachments", setting.Attachment.Path, verbose); err != nil {
-				fatal("Failed to include attachment: %v", err)
-			}
+	if err := storage.Attachments.IterateObjects(func(objPath string, object storage.Object) error {
+		info, err := object.Stat()
+		if err != nil {
+			return err
 		}
-	case setting.MinioStorageType:
-		if err := models.IterateAttachment(func(attach *models.Attachment) error {
-			f, err := storage.Attachments.Open(attach.RelativePath())
-			if err != nil {
-				return err
-			}
-			defer f.Close()
-
-			info, err := f.Stat()
-			if err != nil {
-				return err
-			}
 
-			return w.Write(archiver.File{
-				FileInfo: archiver.FileInfo{
-					FileInfo:   info,
-					CustomName: path.Join("data", "attachments", attach.RelativePath()),
-				},
-				ReadCloser: f,
-			})
-		}); err != nil {
-			fatal("Dump Attachment failed: %v", err)
-		}
-	default:
-		fatal("Unknown Attachment storage type: %s", setting.Attachment.Storage.Type)
+		return w.Write(archiver.File{
+			FileInfo: archiver.FileInfo{
+				FileInfo:   info,
+				CustomName: path.Join("data", "attachments", objPath),
+			},
+			ReadCloser: object,
+		})
+	}); err != nil {
+		fatal("Failed to dump attachments: %v", err)
 	}
 
 	// Doesn't check if LogRootPath exists before processing --skip-log intentionally,

diff --git a/modules/storage/local.go b/modules/storage/local.go
@@ -73,3 +73,28 @@ func (l *LocalStorage) Delete(path string) error {
 func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
 	return nil, ErrURLNotSupported
 }
+
+// IterateObjects iterates across the objects in the local storage
+func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error {
+	return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if path == l.dir {
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+		relPath, err := filepath.Rel(l.dir, path)
+		if err != nil {
+			return err
+		}
+		obj, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer obj.Close()
+		return fn(relPath, obj)
+	})
+}
diff --git a/modules/storage/minio.go b/modules/storage/minio.go
@@ -160,3 +160,24 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
 	reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
 	return m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
 }
+
+// IterateObjects iterates across the objects in the miniostorage
+func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) error {
+	var opts = minio.GetObjectOptions{}
+	lobjectCtx, cancel := context.WithCancel(m.ctx)
+	defer cancel()
+	for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
+		Prefix:    m.basePath,
+		Recursive: true,
+	}) {
+		object, err := m.client.GetObject(lobjectCtx, m.bucket, mObjInfo.Key, opts)
+		if err != nil {
+			return err
+		}
+		defer object.Close()
+		if err = fn(strings.TrimPrefix(m.basePath, mObjInfo.Key), &minioObject{object}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/modules/storage/storage.go b/modules/storage/storage.go
@@ -18,6 +18,8 @@ import (
 var (
 	// ErrURLNotSupported represents url is not supported
 	ErrURLNotSupported = errors.New("url method not supported")
+	// ErrIterateObjectsNotSupported represents IterateObjects not supported
+	ErrIterateObjectsNotSupported = errors.New("iterateObjects method not supported")
 )
 
 // Object represents the object on the storage
@@ -34,6 +36,7 @@ type ObjectStorage interface {
 	Stat(path string) (os.FileInfo, error)
 	Delete(path string) error
 	URL(path, name string) (*url.URL, error)
+	IterateObjects(func(path string, obj Object) error) error
 }
 
 // Copy copys a file from source ObjectStorage to dest ObjectStorage