diff --git a/config/default_config.go b/config/default_config.go index 7ae06c7..e21e0b8 100644 --- a/config/default_config.go +++ b/config/default_config.go @@ -2,7 +2,7 @@ package config var DefaultConfigYaml = ` info: - version: v0.6.1 + version: v0.6.2 edition: global.edition.community mongo: host: localhost diff --git a/config/version.go b/config/version.go index 06188ce..d05606b 100644 --- a/config/version.go +++ b/config/version.go @@ -2,7 +2,7 @@ package config import "strings" -const Version = "v0.6.1" +const Version = "v0.6.2" func GetVersion() (v string) { if strings.HasPrefix(Version, "v") { diff --git a/controllers/spider.go b/controllers/spider.go index 0f74af4..7d5763c 100644 --- a/controllers/spider.go +++ b/controllers/spider.go @@ -77,6 +77,11 @@ func getSpiderActions() []Action { Path: "/:id/files/copy", HandlerFunc: ctx.copyFile, }, + { + Path: "/:id/files/export", + Method: http.MethodPost, + HandlerFunc: ctx.postExport, + }, { Method: http.MethodPost, Path: "/:id/run", @@ -724,6 +729,25 @@ func (ctx *spiderContext) postDataSource(c *gin.Context) { HandleSuccess(c) } +func (ctx *spiderContext) postExport(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // zip file path + zipFilePath, err := ctx.adminSvc.Export(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // download + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) + c.File(zipFilePath) +} + func (ctx *spiderContext) _get(c *gin.Context) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { diff --git a/interfaces/spider_admin_service.go b/interfaces/spider_admin_service.go index fa0a977..c04fc27 100644 --- a/interfaces/spider_admin_service.go +++ b/interfaces/spider_admin_service.go @@ -15,4 +15,6 @@ type SpiderAdminService interface { Delete(id primitive.ObjectID) (err error) // SyncGit syncs the git repository of the spider SyncGit() (err error) + // Export exports the spider and return zip file path + Export(id primitive.ObjectID) (filePath string, err error) } diff --git a/spider/admin/service.go b/spider/admin/service.go index b9625f5..9734f53 100644 --- a/spider/admin/service.go +++ b/spider/admin/service.go @@ -14,11 +14,14 @@ import ( "github.com/crawlab-team/crawlab-core/task/scheduler" "github.com/crawlab-team/crawlab-core/utils" "github.com/crawlab-team/go-trace" + "github.com/google/uuid" "github.com/robfig/cron/v3" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.uber.org/dig" + "os" + "path" "sync" "time" ) @@ -75,6 +78,28 @@ func (svc *Service) SyncGit() (err error) { return nil } +func (svc *Service) Export(id primitive.ObjectID) (filePath string, err error) { + // spider fs + fsSvc, err := fs.NewSpiderFsService(id) + if err != nil { + return "", err + } + + // sync to workspace + if err := fsSvc.GetFsService().SyncToWorkspace(); err != nil { + return "", err + } + + // zip files in workspace + dirPath := fsSvc.GetWorkspacePath() + zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip") + if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil { + return "", trace.TraceError(err) + } + + return zipFilePath, nil +} + func (svc *Service) scheduleTasks(s *models.Spider, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // main task mainTask := &models.Task{ diff --git a/utils/file.go b/utils/file.go index 6bd0a53..39d4254 100644 --- a/utils/file.go +++ b/utils/file.go @@ -384,3 +384,51 @@ func FillEmptyFileData(data []byte) (res []byte) { } return data } + +func ZipDirectory(dir, zipfile string) error { + zipFile, err := os.Create(zipfile) + if err != nil { + return err + } + defer zipFile.Close() + + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + + baseDir := filepath.Dir(dir) + + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(baseDir, path) + if err != nil { + return err + } + + zipFile, err := zipWriter.Create(relPath) + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(zipFile, file) + if err != nil { + return err + } + + return nil + }) + + return err +}