From 58e49e325914ba1f54609347ca3426ab4984c17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 5 Mar 2021 21:08:14 +0100 Subject: [PATCH 1/2] Move api client builders to a cliutil package --- cli/auth.go | 3 +- cli/cmd.go | 266 ++------------------------------- cli/util/api.go | 274 ++++++++++++++++++++++++++++++++++ cmd/lotus-seal-worker/main.go | 3 +- 4 files changed, 287 insertions(+), 259 deletions(-) create mode 100644 cli/util/api.go diff --git a/cli/auth.go b/cli/auth.go index ba20b2bccf6..2f41b38d15f 100644 --- a/cli/auth.go +++ b/cli/auth.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/lotus/api/apistruct" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/node/repo" ) @@ -127,7 +128,7 @@ var authApiInfoToken = &cli.Command{ // TODO: Log in audit log when it is implemented - fmt.Printf("%s=%s:%s\n", envForRepo(t), string(token), ainfo.Addr) + fmt.Printf("%s=%s:%s\n", cliutil.EnvForRepo(t), string(token), ainfo.Addr) return nil }, } diff --git a/cli/cmd.go b/cli/cmd.go index 53ad11bc4d5..4bef6441846 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -1,26 +1,13 @@ package cli import ( - "context" - "fmt" - "net/http" - "net/url" - "os" - "os/signal" "strings" - "syscall" logging "github.com/ipfs/go-log/v2" - "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/api/client" cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/node/repo" ) var log = logging.Logger("cli") @@ -46,167 +33,6 @@ func NewCliError(s string) error { // ApiConnector returns API instance type ApiConnector func() api.FullNode -// The flag passed on the command line with the listen address of the API -// server (only used by the tests) -func flagForAPI(t repo.RepoType) string { - switch t { - case repo.FullNode: - return "api-url" - case repo.StorageMiner: - return "miner-api-url" - case repo.Worker: - return "worker-api-url" - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - -func flagForRepo(t repo.RepoType) string { - switch t { - case repo.FullNode: - return "repo" - case repo.StorageMiner: - return "miner-repo" - case repo.Worker: - return "worker-repo" - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - -func envForRepo(t repo.RepoType) string { - switch t { - case repo.FullNode: - return "FULLNODE_API_INFO" - case repo.StorageMiner: - return "MINER_API_INFO" - case repo.Worker: - return "WORKER_API_INFO" - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - -// TODO remove after deprecation period -func envForRepoDeprecation(t repo.RepoType) string { - switch t { - case repo.FullNode: - return "FULLNODE_API_INFO" - case repo.StorageMiner: - return "STORAGE_API_INFO" - case repo.Worker: - return "WORKER_API_INFO" - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - -func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) { - // Check if there was a flag passed with the listen address of the API - // server (only used by the tests) - apiFlag := flagForAPI(t) - if ctx.IsSet(apiFlag) { - strma := ctx.String(apiFlag) - strma = strings.TrimSpace(strma) - - return cliutil.APIInfo{Addr: strma}, nil - } - - envKey := envForRepo(t) - env, ok := os.LookupEnv(envKey) - if !ok { - // TODO remove after deprecation period - envKey = envForRepoDeprecation(t) - env, ok = os.LookupEnv(envKey) - if ok { - log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, envForRepo(t)) - } - } - if ok { - return cliutil.ParseApiInfo(env), nil - } - - repoFlag := flagForRepo(t) - - p, err := homedir.Expand(ctx.String(repoFlag)) - if err != nil { - return cliutil.APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err) - } - - r, err := repo.NewFS(p) - if err != nil { - return cliutil.APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err) - } - - ma, err := r.APIEndpoint() - if err != nil { - return cliutil.APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err) - } - - token, err := r.APIToken() - if err != nil { - log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err) - } - - return cliutil.APIInfo{ - Addr: ma.String(), - Token: token, - }, nil -} - -func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) { - ainfo, err := GetAPIInfo(ctx, t) - if err != nil { - return "", nil, xerrors.Errorf("could not get API info: %w", err) - } - - addr, err := ainfo.DialArgs() - if err != nil { - return "", nil, xerrors.Errorf("could not get DialArgs: %w", err) - } - - return addr, ainfo.AuthHeader(), nil -} - -func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) { - ti, ok := ctx.App.Metadata["repoType"] - if !ok { - log.Errorf("unknown repo type, are you sure you want to use GetAPI?") - ti = repo.FullNode - } - t, ok := ti.(repo.RepoType) - if !ok { - log.Errorf("repoType type does not match the type of repo.RepoType") - } - - if tn, ok := ctx.App.Metadata["testnode-storage"]; ok { - return tn.(api.StorageMiner), func() {}, nil - } - if tn, ok := ctx.App.Metadata["testnode-full"]; ok { - return tn.(api.FullNode), func() {}, nil - } - - addr, headers, err := GetRawAPI(ctx, t) - if err != nil { - return nil, nil, err - } - - return client.NewCommonRPC(ctx.Context, addr, headers) -} - -func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) { - if tn, ok := ctx.App.Metadata["testnode-full"]; ok { - return tn.(api.FullNode), func() {}, nil - } - - addr, headers, err := GetRawAPI(ctx, repo.FullNode) - if err != nil { - return nil, nil, err - } - - return client.NewFullNodeRPC(ctx.Context, addr, headers) -} - func GetFullNodeServices(ctx *cli.Context) (ServicesAPI, error) { if tn, ok := ctx.App.Metadata["test-services"]; ok { return tn.(ServicesAPI), nil @@ -220,92 +46,18 @@ func GetFullNodeServices(ctx *cli.Context) (ServicesAPI, error) { return &ServicesImpl{api: api, closer: c}, nil } -type GetStorageMinerOptions struct { - PreferHttp bool -} - -type GetStorageMinerOption func(*GetStorageMinerOptions) - -func StorageMinerUseHttp(opts *GetStorageMinerOptions) { - opts.PreferHttp = true -} - -func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) { - var options GetStorageMinerOptions - for _, opt := range opts { - opt(&options) - } - - if tn, ok := ctx.App.Metadata["testnode-storage"]; ok { - return tn.(api.StorageMiner), func() {}, nil - } - - addr, headers, err := GetRawAPI(ctx, repo.StorageMiner) - if err != nil { - return nil, nil, err - } - - if options.PreferHttp { - u, err := url.Parse(addr) - if err != nil { - return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err) - } - - switch u.Scheme { - case "ws": - u.Scheme = "http" - case "wss": - u.Scheme = "https" - } - - addr = u.String() - } - - return client.NewStorageMinerRPC(ctx.Context, addr, headers) -} - -func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.Worker) - if err != nil { - return nil, nil, err - } +var GetAPIInfo = cliutil.GetAPIInfo +var GetRawAPI = cliutil.GetRawAPI +var GetAPI = cliutil.GetAPI - return client.NewWorkerRPC(ctx.Context, addr, headers) -} +var DaemonContext = cliutil.DaemonContext +var ReqContext = cliutil.ReqContext -func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.FullNode) - if err != nil { - return nil, nil, err - } - - return client.NewGatewayRPC(ctx.Context, addr, headers) -} - -func DaemonContext(cctx *cli.Context) context.Context { - if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok { - return mtCtx.(context.Context) - } +var GetFullNodeAPI = cliutil.GetFullNodeAPI +var GetGatewayAPI = cliutil.GetGatewayAPI - return context.Background() -} - -// ReqContext returns context for cli execution. Calling it for the first time -// installs SIGTERM handler that will close returned context. -// Not safe for concurrent execution. -func ReqContext(cctx *cli.Context) context.Context { - tCtx := DaemonContext(cctx) - - ctx, done := context.WithCancel(tCtx) - sigChan := make(chan os.Signal, 2) - go func() { - <-sigChan - done() - }() - signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) - - return ctx -} +var GetStorageMinerAPI = cliutil.GetStorageMinerAPI +var GetWorkerAPI = cliutil.GetWorkerAPI var CommonCommands = []*cli.Command{ netCmd, diff --git a/cli/util/api.go b/cli/util/api.go new file mode 100644 index 00000000000..6a4982894f5 --- /dev/null +++ b/cli/util/api.go @@ -0,0 +1,274 @@ +package cliutil + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/node/repo" +) + +const ( + metadataTraceContext = "traceContext" +) + +// The flag passed on the command line with the listen address of the API +// server (only used by the tests) +func flagForAPI(t repo.RepoType) string { + switch t { + case repo.FullNode: + return "api-url" + case repo.StorageMiner: + return "miner-api-url" + case repo.Worker: + return "worker-api-url" + default: + panic(fmt.Sprintf("Unknown repo type: %v", t)) + } +} + +func flagForRepo(t repo.RepoType) string { + switch t { + case repo.FullNode: + return "repo" + case repo.StorageMiner: + return "miner-repo" + case repo.Worker: + return "worker-repo" + default: + panic(fmt.Sprintf("Unknown repo type: %v", t)) + } +} + +func EnvForRepo(t repo.RepoType) string { + switch t { + case repo.FullNode: + return "FULLNODE_API_INFO" + case repo.StorageMiner: + return "MINER_API_INFO" + case repo.Worker: + return "WORKER_API_INFO" + default: + panic(fmt.Sprintf("Unknown repo type: %v", t)) + } +} + +// TODO remove after deprecation period +func envForRepoDeprecation(t repo.RepoType) string { + switch t { + case repo.FullNode: + return "FULLNODE_API_INFO" + case repo.StorageMiner: + return "STORAGE_API_INFO" + case repo.Worker: + return "WORKER_API_INFO" + default: + panic(fmt.Sprintf("Unknown repo type: %v", t)) + } +} + +func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { + // Check if there was a flag passed with the listen address of the API + // server (only used by the tests) + apiFlag := flagForAPI(t) + if ctx.IsSet(apiFlag) { + strma := ctx.String(apiFlag) + strma = strings.TrimSpace(strma) + + return APIInfo{Addr: strma}, nil + } + + envKey := EnvForRepo(t) + env, ok := os.LookupEnv(envKey) + if !ok { + // TODO remove after deprecation period + envKey = envForRepoDeprecation(t) + env, ok = os.LookupEnv(envKey) + if ok { + log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, EnvForRepo(t)) + } + } + if ok { + return ParseApiInfo(env), nil + } + + repoFlag := flagForRepo(t) + + p, err := homedir.Expand(ctx.String(repoFlag)) + if err != nil { + return APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err) + } + + r, err := repo.NewFS(p) + if err != nil { + return APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err) + } + + ma, err := r.APIEndpoint() + if err != nil { + return APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err) + } + + token, err := r.APIToken() + if err != nil { + log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err) + } + + return APIInfo{ + Addr: ma.String(), + Token: token, + }, nil +} + +func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) { + ainfo, err := GetAPIInfo(ctx, t) + if err != nil { + return "", nil, xerrors.Errorf("could not get API info: %w", err) + } + + addr, err := ainfo.DialArgs() + if err != nil { + return "", nil, xerrors.Errorf("could not get DialArgs: %w", err) + } + + return addr, ainfo.AuthHeader(), nil +} + +func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) { + ti, ok := ctx.App.Metadata["repoType"] + if !ok { + log.Errorf("unknown repo type, are you sure you want to use GetAPI?") + ti = repo.FullNode + } + t, ok := ti.(repo.RepoType) + if !ok { + log.Errorf("repoType type does not match the type of repo.RepoType") + } + + if tn, ok := ctx.App.Metadata["testnode-storage"]; ok { + return tn.(api.StorageMiner), func() {}, nil + } + if tn, ok := ctx.App.Metadata["testnode-full"]; ok { + return tn.(api.FullNode), func() {}, nil + } + + addr, headers, err := GetRawAPI(ctx, t) + if err != nil { + return nil, nil, err + } + + return client.NewCommonRPC(ctx.Context, addr, headers) +} + +func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) { + if tn, ok := ctx.App.Metadata["testnode-full"]; ok { + return tn.(api.FullNode), func() {}, nil + } + + addr, headers, err := GetRawAPI(ctx, repo.FullNode) + if err != nil { + return nil, nil, err + } + + return client.NewFullNodeRPC(ctx.Context, addr, headers) +} + +type GetStorageMinerOptions struct { + PreferHttp bool +} + +type GetStorageMinerOption func(*GetStorageMinerOptions) + +func StorageMinerUseHttp(opts *GetStorageMinerOptions) { + opts.PreferHttp = true +} + +func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) { + var options GetStorageMinerOptions + for _, opt := range opts { + opt(&options) + } + + if tn, ok := ctx.App.Metadata["testnode-storage"]; ok { + return tn.(api.StorageMiner), func() {}, nil + } + + addr, headers, err := GetRawAPI(ctx, repo.StorageMiner) + if err != nil { + return nil, nil, err + } + + if options.PreferHttp { + u, err := url.Parse(addr) + if err != nil { + return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err) + } + + switch u.Scheme { + case "ws": + u.Scheme = "http" + case "wss": + u.Scheme = "https" + } + + addr = u.String() + } + + return client.NewStorageMinerRPC(ctx.Context, addr, headers) +} + +func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) { + addr, headers, err := GetRawAPI(ctx, repo.Worker) + if err != nil { + return nil, nil, err + } + + return client.NewWorkerRPC(ctx.Context, addr, headers) +} + +func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) { + addr, headers, err := GetRawAPI(ctx, repo.FullNode) + if err != nil { + return nil, nil, err + } + + return client.NewGatewayRPC(ctx.Context, addr, headers) +} + +func DaemonContext(cctx *cli.Context) context.Context { + if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok { + return mtCtx.(context.Context) + } + + return context.Background() +} + +// ReqContext returns context for cli execution. Calling it for the first time +// installs SIGTERM handler that will close returned context. +// Not safe for concurrent execution. +func ReqContext(cctx *cli.Context) context.Context { + tCtx := DaemonContext(cctx) + + ctx, done := context.WithCancel(tCtx) + sigChan := make(chan os.Signal, 2) + go func() { + <-sigChan + done() + }() + signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) + + return ctx +} diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 8a17a10a3c6..d5ebf891853 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -31,6 +31,7 @@ import ( "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -183,7 +184,7 @@ var runCmd = &cli.Command{ var closer func() var err error for { - nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, lcli.StorageMinerUseHttp) + nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) if err == nil { _, err = nodeApi.Version(ctx) if err == nil { From a4a21b5b8977edc37510e8f4f2372717da6f386a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 5 Mar 2021 21:33:36 +0100 Subject: [PATCH 2/2] fix lint --- cli/cmd.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cli/cmd.go b/cli/cmd.go index 4bef6441846..19006702f01 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -12,10 +12,6 @@ import ( var log = logging.Logger("cli") -const ( - metadataTraceContext = "traceContext" -) - // custom CLI error type ErrCmdFailed struct {