mody: go mod

refine the code

.gitignore

@@ -23,5 +23,6 @@ dist/
 !app/**/*.js
 tmp
 server/data
+server-v2/api/*/data
 assets/
 bin/

server-v2/api/studio/etc/studio-api.yaml

@@ -8,4 +8,7 @@ Auth:
   AccessSecret: "login_secret"
   AccessExpire: 1800
 File:
-  UploadDir: "./upload/"
+  UploadDir: "./data/upload/"
+  TasksDir: "./data/tasks"
+  SqliteDbFilePath: "./data/tasks.db"
+  TaskIdPath: "./data/taskId.data"

server-v2/api/studio/internal/config/config.go

@@ -2,6 +2,10 @@ package config
 
 import (
 	"github.com/zeromicro/go-zero/rest"
+	"go.uber.org/zap"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 )
 
 type Config struct {
@@ -15,6 +19,76 @@ type Config struct {
 	}
 
 	File struct {
-		UploadDir string
+		UploadDir        string
+		TasksDir         string
+		SqliteDbFilePath string
+		TaskIdPath       string
 	}
 }
+
+const (
+	DefaultFilesDataDir     = "data"
+	DefaultTaskIdPath       = "data/taskId.data"
+	DefaultUploadDir        = "data/upload"
+	DefaultTasksDir         = "data/tasks"
+	DefaultSqlitedbFilePath = "data/tasks.db"
+)
+
+func (c *Config) Validate() error {
+	return nil
+}
+
+func (c *Config) Complete() {
+	if c.File.TaskIdPath == "" {
+		_, err := os.Stat(DefaultFilesDataDir)
+		if os.IsNotExist(err) {
+			os.MkdirAll(DefaultFilesDataDir, 0o766)
+		}
+		abs, _ := filepath.Abs(DefaultTaskIdPath)
+		_, err = ioutil.ReadFile(abs)
+		if err != nil {
+			if os.IsNotExist(err) {
+				_, err := os.Create(abs)
+				if err != nil {
+					zap.L().Fatal("DefaultTaskIdPath Init fail", zap.Error(err))
+				} else {
+					zap.L().Fatal("DefaultTaskIdPath Init fail", zap.Error(err))
+				}
+			}
+		}
+		c.File.TaskIdPath = abs
+	}
+
+	if c.File.UploadDir == "" {
+		abs, _ := filepath.Abs(DefaultTasksDir)
+		c.File.UploadDir = abs
+		_, err := os.Stat(abs)
+		if os.IsNotExist(err) {
+			os.MkdirAll(abs, 0o776)
+		}
+	}
+
+	if c.File.TasksDir == "" {
+		abs, _ := filepath.Abs(DefaultTasksDir)
+		c.File.TasksDir = abs
+		_, err := os.Stat(abs)
+		if os.IsNotExist(err) {
+			os.MkdirAll(abs, 0o766)
+		}
+	}
+
+	if c.File.SqliteDbFilePath == "" {
+		_, err := os.Stat(DefaultFilesDataDir)
+		if os.IsNotExist(err) {
+			os.MkdirAll(DefaultFilesDataDir, 0o766)
+		}
+		abs, _ := filepath.Abs(DefaultSqlitedbFilePath)
+		c.File.SqliteDbFilePath = abs
+	}
+}
+
+func (c *Config) InitConfig() error {
+	c.Complete()
+
+	return c.Validate()
+}

server-v2/api/studio/internal/logic/file/filedestroylogic.go

@@ -26,5 +26,5 @@ func NewFileDestroyLogic(ctx context.Context, svcCtx *svc.ServiceContext) *FileD
 }
 
 func (l *FileDestroyLogic) FileDestroy(req types.FileDestroyRequest) error {
-	return service.NewFileService(nil, l.ctx, l.svcCtx).FileDestroy(req.Name)
+	return service.NewFileService(l.ctx, l.svcCtx).FileDestroy(req.Name)
 }

server-v2/api/studio/internal/logic/file/filesindexlogic.go

@@ -26,5 +26,5 @@ func NewFilesIndexLogic(ctx context.Context, svcCtx *svc.ServiceContext) *FilesI
 }
 
 func (l *FilesIndexLogic) FilesIndex() (resp *types.FilesIndexData, err error) {
-	return service.NewFileService(nil, l.ctx, l.svcCtx).FilesIndex()
+	return service.NewFileService(l.ctx, l.svcCtx).FilesIndex()
 }

server-v2/api/studio/internal/logic/file/fileuploadlogic.go

@@ -1,27 +1,26 @@
 package file
 
 import (
-	"net/http"
-
+	"context"
 	"github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service"
 	"github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc"
 	"github.com/zeromicro/go-zero/core/logx"
 )
 
 type FileUploadLogic struct {
 	logx.Logger
-	r      *http.Request
+	ctx    context.Context
 	svcCtx *svc.ServiceContext
 }
 
-func NewFileUploadLogic(r *http.Request, svcCtx *svc.ServiceContext) *FileUploadLogic {
+func NewFileUploadLogic(ctx context.Context, svcCtx *svc.ServiceContext) *FileUploadLogic {
 	return &FileUploadLogic{
-		Logger: logx.WithContext(r.Context()),
-		r:      r,
+		Logger: logx.WithContext(ctx),
+		ctx:    ctx,
 		svcCtx: svcCtx,
 	}
 }
 
 func (l *FileUploadLogic) FileUpload() error {
-	return service.NewFileService(l.r, nil, l.svcCtx).FileUpload()
+	return service.NewFileService(l.ctx, l.svcCtx).FileUpload()
 }

server-v2/api/studio/internal/service/file.go

@@ -48,22 +48,6 @@ type (
 	}
 )
 
-func NewFileService(r *http.Request, ctx context.Context, svcCtx *svc.ServiceContext) FileService {
-	if r != nil {
-		return &fileService{
-			Logger: logx.WithContext(r.Context()),
-			r:      r,
-			svcCtx: svcCtx,
-		}
-	} else {
-		return &fileService{
-			Logger: logx.WithContext(ctx),
-			ctx:    ctx,
-			svcCtx: svcCtx,
-		}
-	}
-}
-
 func NewFileService(ctx context.Context, svcCtx *svc.ServiceContext) FileService {
 	return &fileService{
 		Logger: logx.WithContext(ctx),

server-v2/api/studio/internal/service/import.go

@@ -8,11 +8,10 @@ import (
 	"fmt"
 	"github.com/vesoft-inc/go-pkg/middleware"
 	importconfig "github.com/vesoft-inc/nebula-importer/pkg/config"
-	importerErrors "github.com/vesoft-inc/nebula-importer/pkg/errors"
+	importererrors "github.com/vesoft-inc/nebula-importer/pkg/errors"
 	"github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/importer"
 	"github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc"
 	"github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types"
-	Config "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/config"
 	"github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode"
 	"github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/utils"
 	"github.com/zeromicro/go-zero/core/logx"
@@ -65,7 +64,7 @@ func NewImportService(ctx context.Context, svcCtx *svc.ServiceContext) ImportSer
 func (i *importService) CreateImportTask(req *types.CreateImportTaskRequest) (*types.CreateImportTaskData, error) {
 	jsons, err := json.Marshal(req.Config)
 	if err != nil {
-		return nil, errors.New("importDataParams get fail")
+		return nil, ecode.WithCode(ecode.ErrParam, nil)
 	}
 
 	conf := importconfig.YAMLConfig{}
@@ -75,20 +74,20 @@ func (i *importService) CreateImportTask(req *types.CreateImportTaskRequest) (*t
 	}
 
 	if err = validClientParams(&conf); err != nil {
-		err = importerErrors.Wrap(importerErrors.InvalidConfigPathOrFormat, err)
+		err = importererrors.Wrap(importererrors.InvalidConfigPathOrFormat, err)
 		zap.L().Warn("client params is wrong", zap.Error(err))
 		return nil, err
 	}
 
-	taskDir, err := importer.GetNewTaskDir()
+	taskDir, err := importer.GetNewTaskDir(i.svcCtx.Config.File.TasksDir)
 	if err != nil {
 		return nil, err
 	}
-	logPath := filepath.Join(taskDir, "import.log")
+	logPath := filepath.Join(taskDir, importLogName)
 	conf.LogPath = &logPath
 
 	// create config file
-	if err := importer.CreateConfigFile(taskDir, conf); err != nil {
+	if err := importer.CreateConfigFile(i.svcCtx.Config.File.UploadDir, taskDir, conf); err != nil {
 		return nil, err
 	}
 
@@ -121,7 +120,7 @@ func (i *importService) CreateImportTask(req *types.CreateImportTaskRequest) (*t
 
 	// write taskId to file
 	muTaskId.Lock()
-	taskIDBytes, err := ioutil.ReadFile(Config.Cfg.Web.TaskIdPath)
+	taskIDBytes, err := ioutil.ReadFile(i.svcCtx.Config.File.TaskIdPath)
 	if err != nil {
 		zap.L().Warn("read taskId file error", zap.Error(err))
 		return nil, err
@@ -138,7 +137,7 @@ func (i *importService) CreateImportTask(req *types.CreateImportTaskRequest) (*t
 	if err != nil {
 		zap.L().Warn("read taskId file error", zap.Error(err))
 	}
-	err = ioutil.WriteFile(Config.Cfg.Web.TaskIdPath, bytes, 777)
+	err = ioutil.WriteFile(i.svcCtx.Config.File.TaskIdPath, bytes, 777)
 	if err != nil {
 		zap.L().Warn("write taskId file error", zap.Error(err))
 	}
@@ -154,10 +153,6 @@ func (i *importService) StopImportTask(req *types.StopImportTaskRequest) error {
 }
 
 func (i *importService) DownloadConfig(req *types.DownloadConfigsRequest) error {
-	if req.Id == "" {
-		return errors.New("invalid Id")
-	}
-
 	httpReq, ok := middleware.GetRequest(i.ctx)
 	if !ok {
 		return ecode.WithInternalServer(fmt.Errorf("unset KeepRequest"))
@@ -168,7 +163,7 @@ func (i *importService) DownloadConfig(req *types.DownloadConfigsRequest) error
 		return ecode.WithInternalServer(fmt.Errorf("unset KeepResponse Writer"))
 	}
 
-	configPath := filepath.Join(Config.Cfg.Web.TasksDir, req.Id, "config.yaml")
+	configPath := filepath.Join(i.svcCtx.Config.File.TasksDir, req.Id, "config.yaml")
 	httpResp.Header().Set("Content-Type", "application/octet-stream")
 	httpResp.Header().Set("Content-Disposition", "attachment;filename="+filepath.Base(configPath))
 	http.ServeFile(httpResp, httpReq, configPath)
@@ -178,9 +173,6 @@ func (i *importService) DownloadConfig(req *types.DownloadConfigsRequest) error
 
 func (i *importService) DownloadLogs(req *types.DownloadLogsRequest) error {
 	id := req.Id
-	if id == "" {
-		return errors.New("id parse failed")
-	}
 
 	httpReq, ok := middleware.GetRequest(i.ctx)
 	if !ok {
@@ -194,41 +186,35 @@ func (i *importService) DownloadLogs(req *types.DownloadLogsRequest) error {
 
 	filename := req.Name
 	path := ""
-	if filename == "import.log" {
-		path = filepath.Join(Config.Cfg.Web.TasksDir, id, filename)
+	if filename == importLogName {
+		path = filepath.Join(i.svcCtx.Config.File.TasksDir, id, filename)
 	} else {
-		path = filepath.Join(Config.Cfg.Web.TasksDir, id, "err", filename)
+		path = filepath.Join(i.svcCtx.Config.File.TasksDir, id, "err", filename)
 	}
 
-	fmt.Println("------------------------")
-	fmt.Println("test")
-
 	httpResp.Header().Set("Content-Type", "application/octet-stream")
 	httpResp.Header().Set("Content-Disposition", "attachment;filename="+filepath.Base(path))
 	http.ServeFile(httpResp, httpReq, path)
 	return nil
 }
 
 func (i *importService) DeleteImportTask(req *types.DeleteImportTaskRequest) error {
-	return importer.DeleteImportTask(req.Id, req.Address+":"+req.Port, req.Username)
+	return importer.DeleteImportTask(i.svcCtx.Config.File.TasksDir, req.Id, req.Address+":"+req.Port, req.Username)
 }
 
 func (i *importService) GetImportTask(req *types.GetImportTaskRequest) (*types.GetImportTaskData, error) {
-	return importer.GetImportTask(req.Id, req.Address+":"+req.Port, req.Username)
+	return importer.GetImportTask(i.svcCtx.Config.File.TasksDir, req.Id, req.Address+":"+req.Port, req.Username)
 }
 
 func (i *importService) GetManyImportTask(req *types.GetManyImportTaskRequest) (*types.GetManyImportTaskData, error) {
-	return importer.GetManyImportTask(req.Address+":"+req.Port, req.Username, req.Page, req.PageSize)
+	return importer.GetManyImportTask(i.svcCtx.Config.File.TasksDir, req.Address+":"+req.Port, req.Username, req.Page, req.PageSize)
 }
 
 // GetImportTaskLogNames :Get all log file's name of a task
 func (i *importService) GetImportTaskLogNames(req *types.GetImportTaskLogNamesRequest) (*types.GetImportTaskLogNamesData, error) {
 	id := req.Id
-	if id == "" {
-		return nil, errors.New("id parse failed")
-	}
 
-	errLogDir := filepath.Join(Config.Cfg.Web.TasksDir, id, "err")
+	errLogDir := filepath.Join(i.svcCtx.Config.File.TasksDir, id, "err")
 	fileInfos, err := ioutil.ReadDir(errLogDir)
 	if err != nil {
 		return nil, err
@@ -237,7 +223,7 @@ func (i *importService) GetImportTaskLogNames(req *types.GetImportTaskLogNamesRe
 	data := &types.GetImportTaskLogNamesData{
 		Names: []string{},
 	}
-	data.Names = append(data.Names, "import.log")
+	data.Names = append(data.Names, importLogName)
 	for _, fileInfo := range fileInfos {
 		name := fileInfo.Name()
 		data.Names = append(data.Names, name)
@@ -248,17 +234,17 @@ func (i *importService) GetImportTaskLogNames(req *types.GetImportTaskLogNamesRe
 func (i *importService) GetManyImportTaskLog(req *types.GetManyImportTaskLogRequest) (*types.GetManyImportTaskLogData, error) {
 	path := ""
 	if req.File == importLogName {
-		path = filepath.Join(Config.Cfg.Web.TasksDir, req.Id, req.File)
+		path = filepath.Join(i.svcCtx.Config.File.TasksDir, req.Id, req.File)
 	} else {
-		path = filepath.Join(Config.Cfg.Web.TasksDir, req.Id, errContentDir, req.File)
+		path = filepath.Join(i.svcCtx.Config.File.TasksDir, req.Id, errContentDir, req.File)
 	}
-	lines, err := readFile(path, req.Offset, req.Limit)
+	lines, err := readFileLines(path, req.Offset, req.Limit)
 	if err != nil {
 		return nil, err
 	}
 
 	muTaskId.RLock()
-	taskIdBytes, err := ioutil.ReadFile(Config.Cfg.Web.TaskIdPath)
+	taskIdBytes, err := ioutil.ReadFile(i.svcCtx.Config.File.TaskIdPath)
 	muTaskId.RUnlock()
 	if err != nil {
 		zap.L().Warn("read taskId file error", zap.Error(err))
@@ -295,7 +281,7 @@ func validClientParams(conf *importconfig.YAMLConfig) error {
 		*conf.NebulaClientSettings.Connection.User == "" ||
 		conf.NebulaClientSettings.Space == nil ||
 		*conf.NebulaClientSettings.Space == "" {
-		return errors.New("client params is wrong")
+		return ecode.WithCode(ecode.ErrParam, nil)
 	}
 
 	for _, fn := range conf.Files {
@@ -308,7 +294,7 @@ func validClientParams(conf *importconfig.YAMLConfig) error {
 	return nil
 }
 
-func readFile(path string, offset int64, limit int64) ([]string, error) {
+func readFileLines(path string, offset int64, limit int64) ([]string, error) {
 	file, err := os.Open(path)
 	if err != nil {
 		zap.L().Warn("open file error", zap.Error(err))