feat: added export spider files

AB#29

config/default_config.go

@@ -2,7 +2,7 @@ package config
 
 var DefaultConfigYaml = `
 info:
-  version: v0.6.1
+  version: v0.6.2
   edition: global.edition.community
 mongo:
   host: localhost

config/version.go

@@ -2,7 +2,7 @@ package config
 
 import "strings"
 
-const Version = "v0.6.1"
+const Version = "v0.6.2"
 
 func GetVersion() (v string) {
 	if strings.HasPrefix(Version, "v") {

controllers/spider.go

@@ -77,6 +77,11 @@ func getSpiderActions() []Action {
 			Path:        "/:id/files/copy",
 			HandlerFunc: ctx.copyFile,
 		},
+		{
+			Path:        "/:id/files/export",
+			Method:      http.MethodPost,
+			HandlerFunc: ctx.postExport,
+		},
 		{
 			Method:      http.MethodPost,
 			Path:        "/:id/run",
@@ -724,6 +729,25 @@ func (ctx *spiderContext) postDataSource(c *gin.Context) {
 	HandleSuccess(c)
 }
 
+func (ctx *spiderContext) postExport(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// zip file path
+	zipFilePath, err := ctx.adminSvc.Export(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// download
+	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath))
+	c.File(zipFilePath)
+}
+
 func (ctx *spiderContext) _get(c *gin.Context) {
 	id, err := primitive.ObjectIDFromHex(c.Param("id"))
 	if err != nil {

interfaces/spider_admin_service.go

@@ -15,4 +15,6 @@ type SpiderAdminService interface {
 	Delete(id primitive.ObjectID) (err error)
 	// SyncGit syncs the git repository of the spider
 	SyncGit() (err error)
+	// Export exports the spider and return zip file path
+	Export(id primitive.ObjectID) (filePath string, err error)
 }

spider/admin/service.go

@@ -14,11 +14,14 @@ import (
 	"github.com/crawlab-team/crawlab-core/task/scheduler"
 	"github.com/crawlab-team/crawlab-core/utils"
 	"github.com/crawlab-team/go-trace"
+	"github.com/google/uuid"
 	"github.com/robfig/cron/v3"
 	"github.com/spf13/viper"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.uber.org/dig"
+	"os"
+	"path"
 	"sync"
 	"time"
 )
@@ -75,6 +78,28 @@ func (svc *Service) SyncGit() (err error) {
 	return nil
 }
 
+func (svc *Service) Export(id primitive.ObjectID) (filePath string, err error) {
+	// spider fs
+	fsSvc, err := fs.NewSpiderFsService(id)
+	if err != nil {
+		return "", err
+	}
+
+	// sync to workspace
+	if err := fsSvc.GetFsService().SyncToWorkspace(); err != nil {
+		return "", err
+	}
+
+	// zip files in workspace
+	dirPath := fsSvc.GetWorkspacePath()
+	zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip")
+	if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil {
+		return "", trace.TraceError(err)
+	}
+
+	return zipFilePath, nil
+}
+
 func (svc *Service) scheduleTasks(s *models.Spider, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) {
 	// main task
 	mainTask := &models.Task{

utils/file.go

@@ -384,3 +384,51 @@ func FillEmptyFileData(data []byte) (res []byte) {
 	}
 	return data
 }
+
+func ZipDirectory(dir, zipfile string) error {
+	zipFile, err := os.Create(zipfile)
+	if err != nil {
+		return err
+	}
+	defer zipFile.Close()
+
+	zipWriter := zip.NewWriter(zipFile)
+	defer zipWriter.Close()
+
+	baseDir := filepath.Dir(dir)
+
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(baseDir, path)
+		if err != nil {
+			return err
+		}
+
+		zipFile, err := zipWriter.Create(relPath)
+		if err != nil {
+			return err
+		}
+
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		_, err = io.Copy(zipFile, file)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	return err
+}