From 9a2796ff9bbe5969cc7f6d16d594ab0a49f6a849 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Tue, 1 May 2018 15:04:08 -0700 Subject: [PATCH 1/5] CHANGELOG-3.4: highlight "CLUSTER_DEBUG" in "integration" Signed-off-by: Gyuho Lee --- CHANGELOG-3.4.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG-3.4.md b/CHANGELOG-3.4.md index 6b8435effb6..dc6d3881419 100644 --- a/CHANGELOG-3.4.md +++ b/CHANGELOG-3.4.md @@ -191,6 +191,11 @@ See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-g - Rename [**`embed.Config.LogOutput`** to **`embed.Config.LogOutputs`**](https://github.com/coreos/etcd/pull/9624) to support multiple log outputs. - Change [**`embed.Config.LogOutputs`** type from `string` to `[]string`](https://github.com/coreos/etcd/pull/9579) to support multiple log outputs. +### Package `integration` + +- Add [`CLUSTER_DEBUG` to enable test cluster logging](https://github.com/coreos/etcd/pull/9678). + - Deprecated `capnslog` in integration tests. + ### API - Add [`snapshot`](https://github.com/coreos/etcd/pull/9118) package for snapshot restore/save operations (see [`godoc.org/github.com/etcd/snapshot`](https://godoc.org/github.com/coreos/etcd/snapshot) for more). From 21d75570ee94f2aae67401cb48d794e2d7136366 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Tue, 1 May 2018 15:05:03 -0700 Subject: [PATCH 2/5] integration: use "zap" logger Signed-off-by: Gyuho Lee --- integration/cluster.go | 98 +++++++++++++++++++++++++++++++++----- integration/logger_test.go | 4 -- 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/integration/cluster.go b/integration/cluster.go index e872e552106..81f47cf72a2 100644 --- a/integration/cluster.go +++ b/integration/cluster.go @@ -51,8 +51,8 @@ import ( "github.com/coreos/etcd/pkg/types" "github.com/coreos/etcd/rafthttp" - "github.com/coreos/pkg/capnslog" "github.com/soheilhy/cmux" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/keepalive" @@ -105,9 +105,15 @@ var ( ClientCertAuth: true, } - plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "integration") + lg = zap.NewNop() ) +func init() { + if os.Getenv("CLUSTER_DEBUG") != "" { + lg, _ = zap.NewProduction() + } +} + type ClusterConfig struct { Size int PeerTLS *transport.TLSInfo @@ -626,6 +632,27 @@ func mustNewMember(t *testing.T, mcfg memberConfig) *member { m.InitialCorruptCheck = true + m.LoggerConfig = &zap.Config{ + Level: zap.NewAtomicLevelAt(zap.InfoLevel), + Development: false, + Sampling: &zap.SamplingConfig{ + Initial: 100, + Thereafter: 100, + }, + Encoding: "json", + EncoderConfig: zap.NewProductionEncoderConfig(), + + OutputPaths: []string{"/dev/null"}, + ErrorOutputPaths: []string{"/dev/null"}, + } + if os.Getenv("CLUSTER_DEBUG") != "" { + m.LoggerConfig.OutputPaths = []string{"stderr"} + m.LoggerConfig.ErrorOutputPaths = []string{"stderr"} + } + m.Logger, err = m.LoggerConfig.Build() + if err != nil { + t.Fatal(err) + } return m } @@ -633,7 +660,7 @@ func mustNewMember(t *testing.T, mcfg memberConfig) *member { func (m *member) listenGRPC() error { // prefix with localhost so cert has right domain m.grpcAddr = "localhost:" + m.Name - if m.useIP { // for IP-only sTLS certs + if m.useIP { // for IP-only TLS certs m.grpcAddr = "127.0.0.1:" + m.Name } l, err := transport.NewUnixListener(m.grpcAddr) @@ -720,7 +747,13 @@ func (m *member) Clone(t *testing.T) *member { // Launch starts a member based on ServerConfig, PeerListeners // and ClientListeners. func (m *member) Launch() error { - plog.Printf("launching %s (%s)", m.Name, m.grpcAddr) + lg.Info( + "launching a member", + zap.String("name", m.Name), + zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), + zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), + zap.String("grpc-address", m.grpcAddr), + ) var err error if m.s, err = etcdserver.NewServer(m.ServerConfig); err != nil { return fmt.Errorf("failed to initialize the etcd server: %v", err) @@ -860,7 +893,13 @@ func (m *member) Launch() error { m.serverClosers = append(m.serverClosers, closer) } - plog.Printf("launched %s (%s)", m.Name, m.grpcAddr) + lg.Info( + "launched a member", + zap.String("name", m.Name), + zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), + zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), + zap.String("grpc-address", m.grpcAddr), + ) return nil } @@ -920,10 +959,22 @@ func (m *member) Close() { // Stop stops the member, but the data dir of the member is preserved. func (m *member) Stop(t *testing.T) { - plog.Printf("stopping %s (%s)", m.Name, m.grpcAddr) + lg.Info( + "stopping a member", + zap.String("name", m.Name), + zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), + zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), + zap.String("grpc-address", m.grpcAddr), + ) m.Close() m.serverClosers = nil - plog.Printf("stopped %s (%s)", m.Name, m.grpcAddr) + lg.Info( + "stopped a member", + zap.String("name", m.Name), + zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), + zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), + zap.String("grpc-address", m.grpcAddr), + ) } // checkLeaderTransition waits for leader transition, returning the new leader ID. @@ -942,7 +993,13 @@ func (m *member) StopNotify() <-chan struct{} { // Restart starts the member using the preserved data dir. func (m *member) Restart(t *testing.T) error { - plog.Printf("restarting %s (%s)", m.Name, m.grpcAddr) + lg.Info( + "restarting a member", + zap.String("name", m.Name), + zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), + zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), + zap.String("grpc-address", m.grpcAddr), + ) newPeerListeners := make([]net.Listener, 0) for _, ln := range m.PeerListeners { newPeerListeners = append(newPeerListeners, NewListenerWithAddr(t, ln.Addr().String())) @@ -961,20 +1018,39 @@ func (m *member) Restart(t *testing.T) error { } err := m.Launch() - plog.Printf("restarted %s (%s)", m.Name, m.grpcAddr) + lg.Info( + "restarted a member", + zap.String("name", m.Name), + zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), + zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), + zap.String("grpc-address", m.grpcAddr), + zap.Error(err), + ) return err } // Terminate stops the member and removes the data dir. func (m *member) Terminate(t *testing.T) { - plog.Printf("terminating %s (%s)", m.Name, m.grpcAddr) + lg.Info( + "terminating a member", + zap.String("name", m.Name), + zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), + zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), + zap.String("grpc-address", m.grpcAddr), + ) m.Close() if !m.keepDataDirTerminate { if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil { t.Fatal(err) } } - plog.Printf("terminated %s (%s)", m.Name, m.grpcAddr) + lg.Info( + "terminated a member", + zap.String("name", m.Name), + zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), + zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), + zap.String("grpc-address", m.grpcAddr), + ) } // Metric gets the metric value for a member diff --git a/integration/logger_test.go b/integration/logger_test.go index 7c6fc424875..48da6e8bdcc 100644 --- a/integration/logger_test.go +++ b/integration/logger_test.go @@ -19,13 +19,9 @@ import ( "github.com/coreos/etcd/clientv3" - "github.com/coreos/pkg/capnslog" "google.golang.org/grpc/grpclog" ) -const defaultLogLevel = capnslog.CRITICAL - func init() { - capnslog.SetGlobalLogLevel(defaultLogLevel) clientv3.SetLogger(grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, ioutil.Discard)) } From f6a14fb72cb54825592a3090ad7c3dee0c26ef82 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Tue, 1 May 2018 15:28:25 -0700 Subject: [PATCH 3/5] clientv3: use "zap" logger in integration tests Signed-off-by: Gyuho Lee --- clientv3/integration/logger_test.go | 2 -- clientv3/ordering/logger_test.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/clientv3/integration/logger_test.go b/clientv3/integration/logger_test.go index bc0220db4b7..48da6e8bdcc 100644 --- a/clientv3/integration/logger_test.go +++ b/clientv3/integration/logger_test.go @@ -19,11 +19,9 @@ import ( "github.com/coreos/etcd/clientv3" - "github.com/coreos/pkg/capnslog" "google.golang.org/grpc/grpclog" ) func init() { - capnslog.SetGlobalLogLevel(capnslog.CRITICAL) clientv3.SetLogger(grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, ioutil.Discard)) } diff --git a/clientv3/ordering/logger_test.go b/clientv3/ordering/logger_test.go index 986c54148ad..798172ec211 100644 --- a/clientv3/ordering/logger_test.go +++ b/clientv3/ordering/logger_test.go @@ -19,11 +19,9 @@ import ( "github.com/coreos/etcd/clientv3" - "github.com/coreos/pkg/capnslog" "google.golang.org/grpc/grpclog" ) func init() { - capnslog.SetGlobalLogLevel(capnslog.CRITICAL) clientv3.SetLogger(grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, ioutil.Discard)) } From 38326d002ed90da08f9670c24c9dd7ae1a2eb3c3 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Tue, 1 May 2018 15:28:54 -0700 Subject: [PATCH 4/5] etcdserver/v2store: use "zap" logger in v2v3 tests Signed-off-by: Gyuho Lee --- etcdserver/v2store/store_v2v3_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/etcdserver/v2store/store_v2v3_test.go b/etcdserver/v2store/store_v2v3_test.go index 8232ce09a30..4d2703c40fb 100644 --- a/etcdserver/v2store/store_v2v3_test.go +++ b/etcdserver/v2store/store_v2v3_test.go @@ -25,12 +25,10 @@ import ( "github.com/coreos/etcd/etcdserver/v2store" "github.com/coreos/etcd/integration" - "github.com/coreos/pkg/capnslog" "google.golang.org/grpc/grpclog" ) func init() { - capnslog.SetGlobalLogLevel(capnslog.CRITICAL) clientv3.SetLogger(grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, ioutil.Discard)) } From 6cf3dae93e138255c93b7995b4a07307b986817f Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Tue, 1 May 2018 16:19:55 -0700 Subject: [PATCH 5/5] etcdserver/api/v3rpc: fix race in stream error logging Signed-off-by: Gyuho Lee --- etcdserver/api/v3rpc/watch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etcdserver/api/v3rpc/watch.go b/etcdserver/api/v3rpc/watch.go index 643d1dab94e..5a1f621cc2d 100644 --- a/etcdserver/api/v3rpc/watch.go +++ b/etcdserver/api/v3rpc/watch.go @@ -159,7 +159,7 @@ func (ws *watchServer) Watch(stream pb.Watch_WatchServer) (err error) { if rerr := sws.recvLoop(); rerr != nil { if isClientCtxErr(stream.Context().Err(), rerr) { if sws.lg != nil { - sws.lg.Debug("failed to receive watch request from gRPC stream", zap.Error(err)) + sws.lg.Debug("failed to receive watch request from gRPC stream", zap.Error(rerr)) } else { plog.Debugf("failed to receive watch request from gRPC stream (%q)", rerr.Error()) }