From aca6d6ba3a69621ce4a1e959490467c8be43eaf8 Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 16:16:55 +0800 Subject: [PATCH 01/13] test dead link checker Signed-off-by: seeflood --- docs/zh/start/trace/trace.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/start/trace/trace.md b/docs/zh/start/trace/trace.md index a9c6c849c6..04c97cc77c 100644 --- a/docs/zh/start/trace/trace.md +++ b/docs/zh/start/trace/trace.md @@ -170,7 +170,7 @@ curl --location --request GET 'http://127.0.0.1:34903/metrics' 解释一下[runtime_config.json](https://github.com/mosn/layotto/blob/main/configs/runtime_config.json) 里 metrics 相关配置 #### 埋点、统计 -![](https://user-images.githubusercontent.com/26001097/151318373-632e93bc-108d-47ae-b401-6092ed66bcdc.png) + 图中标红的这段配置会启用mosn的"grpc_metric" filter。这个filter的作用是在每次处理完grpc请求后，统计服务名、成功还是失败等信息，存在内存中。 From be711159653fdcdb174396bbc79f467a00c0f65c Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 17:51:20 +0800 Subject: [PATCH 02/13] errcheck & forcetypeassert Signed-off-by: seeflood --- .golangci.yml | 2 ++ pkg/actuator/health/endpoint_test.go | 3 ++- pkg/grpc/default_api/api_file_test.go | 3 ++- pkg/runtime/runtime_test.go | 3 ++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index fabaddf08c..c1d300c320 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -29,6 +29,8 @@ linters : - gocyclo - revive - goimports + - errcheck + - forcetypeassert # - lll issues : diff --git a/pkg/actuator/health/endpoint_test.go b/pkg/actuator/health/endpoint_test.go index d376bda149..e286d5f301 100644 --- a/pkg/actuator/health/endpoint_test.go +++ b/pkg/actuator/health/endpoint_test.go @@ -68,7 +68,8 @@ func TestEndpoint_WhenNoIndicator(t *testing.T) { assert.True(t, err != nil) assert.True(t, len(handle) == 2) assert.True(t, handle["status"] == DOWN) - health := handle["components"].(map[string]Health)["test"] + health, ok := handle["components"].(map[string]Health)["test"] + assert.True(t, ok) assert.True(t, health.Status == DOWN) assert.True(t, health.GetDetail("reason") == "mock") } diff --git a/pkg/grpc/default_api/api_file_test.go b/pkg/grpc/default_api/api_file_test.go index fee088d156..14b483728a 100644 --- a/pkg/grpc/default_api/api_file_test.go +++ b/pkg/grpc/default_api/api_file_test.go @@ -51,7 +51,8 @@ func TestGetFile(t *testing.T) { mockStream.EXPECT().Send(&runtimev1pb.GetFileResponse{Data: []byte("testFile")}).Times(1) mockStream.EXPECT().Context().Return(context.Background()) go SendData(w) - api.GetFile(&runtimev1pb.GetFileRequest{StoreName: "mock"}, mockStream) + err = api.GetFile(&runtimev1pb.GetFileRequest{StoreName: "mock"}, mockStream) + assert.Nil(t, err) } func TestPutFile(t *testing.T) { diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index 9dee49090d..9fe57ddf4b 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -493,7 +493,8 @@ func TestMosnRuntime_initOutputBinding(t *testing.T) { m.runtimeConfig.Bindings["mockOutbindings"] = mbindings.Metadata{ Metadata: mdata, } - m.initOutputBinding(registry) + err := m.initOutputBinding(registry) + assert.Nil(t, err) assert.NotNil(t, m.outputBindings["mockOutbindings"]) } From 7dfff986bff71a195fc97a4592e4076c5992dce5 Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 18:14:04 +0800 Subject: [PATCH 03/13] check go.mod Signed-off-by: seeflood --- make/golang.mk | 4 ++++ spec/go.mod | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/make/golang.mk b/make/golang.mk index d995eba4bb..90434f04bf 100644 --- a/make/golang.mk +++ b/make/golang.mk @@ -152,3 +152,7 @@ go.format: go.format.verify $(GO_FMT) -s -w . $(GOPATH)/bin/$(GO_IMPORTS) -w -local $(GO_MODULE) . $(GO) mod tidy + cd components && $(GO) mod tidy + cd demo && $(GO) mod tidy + cd sdk/go-sdk && $(GO) mod tidy + cd spec && $(GO) mod tidy diff --git a/spec/go.mod b/spec/go.mod index 9abdeb39fe..61ce5343f6 100644 --- a/spec/go.mod +++ b/spec/go.mod @@ -3,7 +3,7 @@ module mosn.io/layotto/spec go 1.14 require ( - github.com/golang/protobuf v1.5.0 + github.com/golang/protobuf v1.5.0 // indirect google.golang.org/grpc v1.37.0 google.golang.org/protobuf v1.26.0-rc.1 ) From 4a1d376ea6268cc08f691f68d33d229005270bbf Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 18:14:23 +0800 Subject: [PATCH 04/13] wrong go.sum for test Signed-off-by: seeflood --- components/go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/go.sum b/components/go.sum index cb0e47f59b..1eda13d7db 100644 --- a/components/go.sum +++ b/components/go.sum @@ -6,6 +6,8 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= From 95b1bd19c42af1eb98c0c5c27892f22b85d799e1 Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 18:30:00 +0800 Subject: [PATCH 05/13] wrong go.sum for test Signed-off-by: seeflood --- .golangci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index c1d300c320..e6c7e98384 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -29,8 +29,8 @@ linters : - gocyclo - revive - goimports - - errcheck - - forcetypeassert +# - errcheck +# - forcetypeassert # - lll issues : From 6f9bb8f65d3782287720f85c6645ea254b59d0a7 Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 20:50:44 +0800 Subject: [PATCH 06/13] fix linter issues in hdfs code Signed-off-by: seeflood --- components/file/hdfs/hdfs.go | 5 +---- components/file/hdfs/hdfs_test.go | 30 +++++++++++++++++------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/components/file/hdfs/hdfs.go b/components/file/hdfs/hdfs.go index bb7f7447c7..f89e69ee0d 100644 --- a/components/file/hdfs/hdfs.go +++ b/components/file/hdfs/hdfs.go @@ -268,8 +268,5 @@ func (h *hdfs) createHdfsClient(meta *HdfsMetaData) (types.Storager, error) { // ishdfsMetaValid check if the metadata is valid func (hm *HdfsMetaData) isHdfsMetaValid() bool { - if hm.EndPoint == "" { - return false - } - return true + return hm.EndPoint != "" } diff --git a/components/file/hdfs/hdfs_test.go b/components/file/hdfs/hdfs_test.go index e6bc0fcd7c..35ee6b1d39 100644 --- a/components/file/hdfs/hdfs_test.go +++ b/components/file/hdfs/hdfs_test.go @@ -29,12 +29,16 @@ import ( "go.beyondstorage.io/v5/types" ) -// config is the raw json data of component's Metadata configuration -const config = `[ +const ( + // config is the raw json data of component's Metadata configuration + config = `[ { "endpoint": "tcp:127.0.0.1:9000" } ]` + endpoint9000 = "127.0.0.1:9000" + endpoint9000TCP = "tcp:127.0.0.1:9000" +) func TestHdfs_Init(t *testing.T) { hdfs := NewHdfs() @@ -63,7 +67,7 @@ func TestHdfs_selectClient(t *testing.T) { assert.Equal(t, err, ErrInitFailed) meta := make(map[string]string) - meta["endpoint"] = "tcp:127.0.0.1:9000" + meta["endpoint"] = endpoint9000TCP _, err = hdfs.selectClient(meta) assert.NotNil(t, err) @@ -112,7 +116,7 @@ func TestHdfs_Put(t *testing.T) { assert.NotNil(t, err) // convert from string to int64 success - req.Metadata["endpoint"] = "tcp:127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000TCP req.Metadata["fileSize"] = "123" err = hdfs.Put(context.TODO(), req) assert.NotNil(t, err) @@ -138,11 +142,11 @@ func TestHdfs_Get(t *testing.T) { assert.Equal(t, ErrMissingEndPoint, err) // client not exist - req.Metadata["endpoint"] = "127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000 _, err = hdfs.Get(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) - req.Metadata["endpoint"] = "tcp:127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000TCP _, err = hdfs.Get(context.TODO(), req) assert.NotNil(t, err) @@ -168,11 +172,11 @@ func TestHdfs_Del(t *testing.T) { assert.Equal(t, ErrMissingEndPoint, err) // client not exist - req.Metadata["endpoint"] = "127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000 err = hdfs.Del(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) - req.Metadata["endpoint"] = "tcp:127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000TCP err = hdfs.Del(context.TODO(), req) assert.NotNil(t, err) } @@ -198,11 +202,11 @@ func TestHdfs_List(t *testing.T) { assert.Equal(t, ErrMissingEndPoint, err) assert.Nil(t, resp) - req.Metadata["endpoint"] = "127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000 resp, err = hdfs.List(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) - req.Metadata["endpoint"] = "tcp:127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000TCP resp, err = hdfs.List(context.TODO(), req) assert.NotNil(t, err) } @@ -227,11 +231,11 @@ func TestHdfs_Stat(t *testing.T) { assert.Equal(t, ErrNotSpecifyEndpoint, err) assert.Nil(t, resp) - req.Metadata["endpoint"] = "127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000 resp, err = hdfs.Stat(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) - req.Metadata["endpoint"] = "tcp:127.0.0.1:9000" + req.Metadata["endpoint"] = endpoint9000TCP resp, err = hdfs.Stat(context.TODO(), req) assert.NotNil(t, err) } @@ -253,7 +257,7 @@ func TestHdfs_CreateHdfsClient(t *testing.T) { assert.Nil(t, store) assert.Error(t, err) - mt.EndPoint = "tcp:127.0.0.1:9000" + mt.EndPoint = endpoint9000TCP store, err = oss.(*hdfs).createHdfsClient(mt) assert.Nil(t, store) assert.Error(t, err) From 5add88db824cb51dc38c8b92c01bcfa7ba32fb1c Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 20:51:14 +0800 Subject: [PATCH 07/13] go.style Signed-off-by: seeflood --- make/golang.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/golang.mk b/make/golang.mk index 90434f04bf..ffb41f4d54 100644 --- a/make/golang.mk +++ b/make/golang.mk @@ -114,7 +114,7 @@ go.test: go.test.verify .PHONY: go.style go.style: @echo "===========> Running go style check" - $(GO) fmt ./... && git status && [[ -z `git status -s` ]] + $(MAKE) format && git status && [[ -z `git status -s` ]] .PHONY: go.format.verify go.format.verify: From 1b89f72ecb6e76b7171ee1cc92d7e057230932f9 Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 20:54:20 +0800 Subject: [PATCH 08/13] fix go.sum Signed-off-by: seeflood --- components/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/go.sum b/components/go.sum index 1eda13d7db..cb0e47f59b 100644 --- a/components/go.sum +++ b/components/go.sum @@ -6,8 +6,6 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= From 89c78dba4ff0a10f972b22f44cc16e79f5562b7c Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 20:57:48 +0800 Subject: [PATCH 09/13] ignore goimports Signed-off-by: seeflood --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6ac94b0225..9c481fdce6 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,9 @@ application.pid # integrate test integrate_test.sh +# production of make script +goimports + # production of quickstart testing cmd/layotto/layotto cmd/layotto/nohup.out From 2f1215a0c8bb822ecec1be7ff0a891479bdcf258 Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 21:14:29 +0800 Subject: [PATCH 10/13] fix links Signed-off-by: seeflood --- docs/zh/start/trace/trace.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/start/trace/trace.md b/docs/zh/start/trace/trace.md index 04c97cc77c..1ce554752d 100644 --- a/docs/zh/start/trace/trace.md +++ b/docs/zh/start/trace/trace.md @@ -170,7 +170,7 @@ curl --location --request GET 'http://127.0.0.1:34903/metrics' 解释一下[runtime_config.json](https://github.com/mosn/layotto/blob/main/configs/runtime_config.json) 里 metrics 相关配置 #### 埋点、统计 - + 图中标红的这段配置会启用mosn的"grpc_metric" filter。这个filter的作用是在每次处理完grpc请求后，统计服务名、成功还是失败等信息，存在内存中。 From 0a0e206627a2473e613fb5427e549bdb57b2e40d Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 22:04:17 +0800 Subject: [PATCH 11/13] modify hdfs Signed-off-by: seeflood --- components/file/hdfs/hdfs.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/file/hdfs/hdfs.go b/components/file/hdfs/hdfs.go index f89e69ee0d..73365e9313 100644 --- a/components/file/hdfs/hdfs.go +++ b/components/file/hdfs/hdfs.go @@ -258,12 +258,7 @@ func (h *hdfs) selectClient(meta map[string]string) (client types.Storager, err } func (h *hdfs) createHdfsClient(meta *HdfsMetaData) (types.Storager, error) { - client, err := store.NewStorager(pairs.WithEndpoint(meta.EndPoint)) - - if err != nil { - return nil, err - } - return client, nil + return store.NewStorager(pairs.WithEndpoint(meta.EndPoint)) } // ishdfsMetaValid check if the metadata is valid From 73f9f045240b23fd7f422fb74079179d30c71d2a Mon Sep 17 00:00:00 2001 From: seeflood Date: Mon, 23 May 2022 22:05:12 +0800 Subject: [PATCH 12/13] modify hdfs Signed-off-by: seeflood --- components/file/hdfs/hdfs.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/file/hdfs/hdfs.go b/components/file/hdfs/hdfs.go index 73365e9313..5244ef2773 100644 --- a/components/file/hdfs/hdfs.go +++ b/components/file/hdfs/hdfs.go @@ -109,12 +109,7 @@ func (h *hdfs) Put(ctx context.Context, stu *file.PutFileStu) error { } _, err = client.Write(stu.FileName, stu.DataStream, size) - - if err != nil { - return err - } - - return nil + return err } func (h *hdfs) Get(ctx context.Context, stu *file.GetFileStu) (io.ReadCloser, error) { From ec14ff2c31fec62cd1aad9e98cc9ea3f99386c65 Mon Sep 17 00:00:00 2001 From: seeflood Date: Tue, 24 May 2022 12:07:35 +0800 Subject: [PATCH 13/13] rename Signed-off-by: seeflood --- components/file/hdfs/hdfs_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/components/file/hdfs/hdfs_test.go b/components/file/hdfs/hdfs_test.go index 35ee6b1d39..cc166e294a 100644 --- a/components/file/hdfs/hdfs_test.go +++ b/components/file/hdfs/hdfs_test.go @@ -36,8 +36,8 @@ const ( "endpoint": "tcp:127.0.0.1:9000" } ]` - endpoint9000 = "127.0.0.1:9000" - endpoint9000TCP = "tcp:127.0.0.1:9000" + endpoint = "127.0.0.1:9000" + tcpEndpoint = "tcp:127.0.0.1:9000" ) func TestHdfs_Init(t *testing.T) { @@ -67,7 +67,7 @@ func TestHdfs_selectClient(t *testing.T) { assert.Equal(t, err, ErrInitFailed) meta := make(map[string]string) - meta["endpoint"] = endpoint9000TCP + meta["endpoint"] = tcpEndpoint _, err = hdfs.selectClient(meta) assert.NotNil(t, err) @@ -116,7 +116,7 @@ func TestHdfs_Put(t *testing.T) { assert.NotNil(t, err) // convert from string to int64 success - req.Metadata["endpoint"] = endpoint9000TCP + req.Metadata["endpoint"] = tcpEndpoint req.Metadata["fileSize"] = "123" err = hdfs.Put(context.TODO(), req) assert.NotNil(t, err) @@ -142,11 +142,11 @@ func TestHdfs_Get(t *testing.T) { assert.Equal(t, ErrMissingEndPoint, err) // client not exist - req.Metadata["endpoint"] = endpoint9000 + req.Metadata["endpoint"] = endpoint _, err = hdfs.Get(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) - req.Metadata["endpoint"] = endpoint9000TCP + req.Metadata["endpoint"] = tcpEndpoint _, err = hdfs.Get(context.TODO(), req) assert.NotNil(t, err) @@ -172,11 +172,11 @@ func TestHdfs_Del(t *testing.T) { assert.Equal(t, ErrMissingEndPoint, err) // client not exist - req.Metadata["endpoint"] = endpoint9000 + req.Metadata["endpoint"] = endpoint err = hdfs.Del(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) - req.Metadata["endpoint"] = endpoint9000TCP + req.Metadata["endpoint"] = tcpEndpoint err = hdfs.Del(context.TODO(), req) assert.NotNil(t, err) } @@ -202,11 +202,11 @@ func TestHdfs_List(t *testing.T) { assert.Equal(t, ErrMissingEndPoint, err) assert.Nil(t, resp) - req.Metadata["endpoint"] = endpoint9000 + req.Metadata["endpoint"] = endpoint resp, err = hdfs.List(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) - req.Metadata["endpoint"] = endpoint9000TCP + req.Metadata["endpoint"] = tcpEndpoint resp, err = hdfs.List(context.TODO(), req) assert.NotNil(t, err) } @@ -231,11 +231,11 @@ func TestHdfs_Stat(t *testing.T) { assert.Equal(t, ErrNotSpecifyEndpoint, err) assert.Nil(t, resp) - req.Metadata["endpoint"] = endpoint9000 + req.Metadata["endpoint"] = endpoint resp, err = hdfs.Stat(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) - req.Metadata["endpoint"] = endpoint9000TCP + req.Metadata["endpoint"] = tcpEndpoint resp, err = hdfs.Stat(context.TODO(), req) assert.NotNil(t, err) } @@ -257,7 +257,7 @@ func TestHdfs_CreateHdfsClient(t *testing.T) { assert.Nil(t, store) assert.Error(t, err) - mt.EndPoint = endpoint9000TCP + mt.EndPoint = tcpEndpoint store, err = oss.(*hdfs).createHdfsClient(mt) assert.Nil(t, store) assert.Error(t, err)