Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TestKeyManager test panicked on 10 minutes timeout #4879

Closed
JmPotato opened this issue Apr 29, 2022 · 1 comment
Closed

TestKeyManager test panicked on 10 minutes timeout #4879

JmPotato opened this issue Apr 29, 2022 · 1 comment
Labels
type/ci The issue is related to CI.

Comments

@JmPotato
Copy link
Member

Flaky Test

Which jobs are failing

FAIL	github.com/tikv/pd/server/encryptionkm	600.328s

CI link

https://github.com/tikv/pd/runs/6207105310?check_suite_focus=true

Reason for failure (if possible)

panic: test timed out after 10m0s

goroutine 1542 [running]:
testing.(*M).startAlarm.func1()
	/opt/hostedtoolcache/go/1.18.0/x64/src/testing/testing.go:2029 +0xbb
created by time.goFunc
	/opt/hostedtoolcache/go/1.18.0/x64/src/time/sleep.go:176 +0x48

goroutine 1 [chan receive, 10 minutes]:
testing.(*T).Run(0xc0000e44e0, {0x2d40e4d, 0xe}, 0x3675770)
	/opt/hostedtoolcache/go/1.18.0/x64/src/testing/testing.go:1487 +0x750
testing.runTests.func1(0x0?)
	/opt/hostedtoolcache/go/1.18.0/x64/src/testing/testing.go:1839 +0x9a
testing.tRunner(0xc0000e44e0, 0xc0005bdb68)
	/opt/hostedtoolcache/go/1.18.0/x64/src/testing/testing.go:1439 +0x214
testing.runTests(0xc0004e40a0?, {0x4c07380, 0x1, 0x1}, {0x40?, 0x7fdf4f6c9f20?, 0x4d2e000?})
	/opt/hostedtoolcache/go/1.18.0/x64/src/testing/testing.go:1837 +0x7e5
testing.(*M).Run(0xc0004e40a0)
	/opt/hostedtoolcache/go/1.18.0/x64/src/testing/testing.go:1719 +0xa72
main.main()
	_testmain.go:975 +0x3ca

goroutine 1519 [select, 9 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0017dd4a0, 0x1)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:395 +0x1ad
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00130fe00)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:513 +0x11b
google.golang.org/grpc/internal/transport.newHTTP2Server.func2()
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:296 +0x1b5
created by google.golang.org/grpc/internal/transport.newHTTP2Server
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:293 +0x22a9

goroutine 1532 [select, 9 minutes]:
context.propagateCancel.func1()
	/opt/hostedtoolcache/go/1.18.0/x64/src/context/context.go:279 +0xdc
created by context.propagateCancel
	/opt/hostedtoolcache/go/1.18.0/x64/src/context/context.go:278 +0x265

goroutine 1540 [select, 9 minutes]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0000aa3c0, {0xc001284880, 0x5, 0x5})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:174 +0x133
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000aa3c0, {0xc001284880, 0x5, 0x5})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:168 +0x3dc
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012f6960, {0xc001284880, 0x5, 0x5})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:479 +0x66
io.ReadAtLeast({0x3fd9ec0, 0xc0012f6960}, {0xc001284880, 0x5, 0x5}, 0x5)
	/opt/hostedtoolcache/go/1.18.0/x64/src/io/io.go:331 +0xde
io.ReadFull(...)
	/opt/hostedtoolcache/go/1.18.0/x64/src/io/io.go:350
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000170500, {0xc001284880, 0x5, 0x5})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:463 +0x125
google.golang.org/grpc.(*parser).recvMsg(0xc001284870, 0x200000)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:508 +0x75
google.golang.org/grpc.recvAndDecompress(0x50?, 0xc000170500, {0x0, 0x0}, 0x200000, 0x0, {0x0, 0x0})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:639 +0x88
google.golang.org/grpc.recv(0x203000?, {0x7fdf247e3cc8, 0x577e620}, 0x7fdf4fcc9008?, {0x0, 0x0}, {0x2c8d7c0, 0xc0002cfa50}, 0x7fdf1d630008?, 0x0, ...)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:707 +0xba
google.golang.org/grpc.(*serverStream).RecvMsg(0xc00200a0c0, {0x2c8d7c0?, 0xc0002cfa50})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:1474 +0x2b2
github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredServerStream).RecvMsg(0xc0012848a0, {0x2c8d7c0, 0xc0002cfa50})
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:164 +0x59
go.etcd.io/etcd/etcdserver/etcdserverpb.(*watchWatchServer).Recv(0xc0012b3a30)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/etcdserverpb/rpc.pb.go:3743 +0x68
go.etcd.io/etcd/etcdserver/api/v3rpc.(*serverWatchStream).recvLoop(0xc00200a180)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/watch.go:222 +0x67
go.etcd.io/etcd/etcdserver/api/v3rpc.(*watchServer).Watch.func2()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/watch.go:173 +0x6a
created by go.etcd.io/etcd/etcdserver/api/v3rpc.(*watchServer).Watch
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/watch.go:172 +0x4cf

goroutine 1423 [select]:
go.etcd.io/etcd/lease.(*lessor).runLoop(0xc0003c4f20)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/lease/lessor.go:590 +0x195
created by go.etcd.io/etcd/lease.newLessor
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/lease/lessor.go:222 +0x57e

goroutine 1535 [select, 9 minutes]:
google.golang.org/grpc.newClientStream.func5()
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:319 +0xec
created by google.golang.org/grpc.newClientStream
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:318 +0x1eaa

goroutine 1514 [select, 9 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0011520c0)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/balancer_conn_wrappers.go:69 +0x156
created by google.golang.org/grpc.newCCBalancerWrapper
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/balancer_conn_wrappers.go:60 +0x3b2

goroutine 1510 [select]:
go.etcd.io/etcd/pkg/fileutil.purgeFile.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/fileutil/purge.go:80 +0x257
created by go.etcd.io/etcd/pkg/fileutil.purgeFile
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/fileutil/purge.go:34 +0x24f

goroutine 1539 [select, 9 minutes]:
go.etcd.io/etcd/etcdserver/api/v3rpc.(*serverWatchStream).sendLoop(0xc00200a180)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/watch.go:359 +0x34f
go.etcd.io/etcd/etcdserver/api/v3rpc.(*watchServer).Watch.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/watch.go:163 +0x31
created by go.etcd.io/etcd/etcdserver/api/v3rpc.(*watchServer).Watch
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/watch.go:162 +0x3c9

goroutine 1537 [select, 9 minutes]:
go.etcd.io/etcd/clientv3.(*watchGrpcStream).serveSubstream(0xc001158000, 0xc0002e9290, 0xc001156b40)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/clientv3/watch.go:762 +0x3e6
created by go.etcd.io/etcd/clientv3.(*watchGrpcStream).run
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/clientv3/watch.go:541 +0x973

goroutine 70 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc0000e64b0)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/logutil/merge_logger.go:173 +0xb0
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/logutil/merge_logger.go:91 +0x157

goroutine 89 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc0000e73c8)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/logutil/merge_logger.go:173 +0xb0
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/logutil/merge_logger.go:91 +0x157

goroutine 91 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc0000e74d0)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/logutil/merge_logger.go:173 +0xb0
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/logutil/merge_logger.go:91 +0x157

goroutine 1497 [select]:
go.etcd.io/etcd/etcdserver.(*raftNode).start.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/raft.go:170 +0x259
created by go.etcd.io/etcd/etcdserver.(*raftNode).start
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/raft.go:165 +0xed

goroutine 198 [select, 9 minutes]:
github.com/pingcap/check.(*resultTracker)._loopRoutine(0xc0003a01b0)
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/check.go:464 +0x153
created by github.com/pingcap/check.(*resultTracker).start
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/check.go:444 +0x8f

goroutine 197 [select, 9 minutes]:
github.com/pingcap/check.(*suiteRunner).doRun(0xc0003a02d0, 0x0)
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/check.go:667 +0x2a5
github.com/pingcap/check.(*suiteRunner).run(0xc0003a02d0)
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/check.go:696 +0xe6
github.com/pingcap/check.Run({0x2cbef00, 0x577e620}, 0xc0001379c0?)
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/run.go:150 +0x4a
github.com/pingcap/check.RunAll(0xc00007bd00?)
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/run.go:117 +0x259
github.com/pingcap/check.TestingT(0xc0000e4680)
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/run.go:99 +0x647
github.com/tikv/pd/server/encryptionkm.TestKeyManager(0x0?)
	/home/runner/work/pd/pd/server/encryptionkm/key_manager_test.go:41 +0x2f
testing.tRunner(0xc0000e4680, 0x3675770)
	/opt/hostedtoolcache/go/1.18.0/x64/src/testing/testing.go:1439 +0x214
created by testing.(*T).Run
	/opt/hostedtoolcache/go/1.18.0/x64/src/testing/testing.go:1486 +0x725

goroutine 1533 [select, 9 minutes]:
go.etcd.io/etcd/clientv3.(*watchGrpcStream).run(0xc001158000)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/clientv3/watch.go:524 +0x459
created by go.etcd.io/etcd/clientv3.(*watcher).newWatcherGrpcStream
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/clientv3/watch.go:277 +0x645

goroutine 1482 [select, 9 minutes]:
go.etcd.io/etcd/etcdserver.(*EtcdServer).purgeFile(0xc000390b00)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:819 +0x44f
go.etcd.io/etcd/etcdserver.(*EtcdServer).goAttach.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2632 +0x7b
created by go.etcd.io/etcd/etcdserver.(*EtcdServer).goAttach
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2630 +0x2a6

goroutine 1506 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x7fdf4f681f88, 0x72)
	/opt/hostedtoolcache/go/1.18.0/x64/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc00181c318, 0x1?, 0x0)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_poll_runtime.go:83 +0xbd
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Accept(0xc00181c300)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_unix.go:614 +0x425
net.(*netFD).accept(0xc00181c300)
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/fd_unix.go:172 +0x4a
net.(*TCPListener).accept(0xc0003073f8)
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/tcpsock_posix.go:139 +0x45
net.(*TCPListener).Accept(0xc0003073f8)
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/tcpsock.go:288 +0x68
go.etcd.io/etcd/pkg/transport.(*rwTimeoutListener).Accept(0xc001337b00)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/transport/timeout_listener.go:48 +0x45
github.com/soheilhy/cmux.(*cMux).Serve(0xc0017dc4b0)
	/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:162 +0x122
go.etcd.io/etcd/embed.(*Etcd).servePeers.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/etcd.go:549 +0x42
go.etcd.io/etcd/embed.(*Etcd).servePeers.func3(0xc001337ae0)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/etcd.go:583 +0x2a9
created by go.etcd.io/etcd/embed.(*Etcd).servePeers
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/etcd.go:573 +0xd25

goroutine 1538 [select, 9 minutes]:
go.etcd.io/etcd/etcdserver/api/v3rpc.(*watchServer).Watch(0xc00184caa0, {0x3ff3380?, 0xc0012b3a30})
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/watch.go:192 +0x570
go.etcd.io/etcd/etcdserver/etcdserverpb._Watch_Watch_Handler({0x2af1a20?, 0xc00184caa0}, {0x3ff0f48?, 0xc0012848a0})
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/etcdserverpb/rpc.pb.go:3724 +0xbf
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1({0x2af1a20, 0xc00184caa0}, {0x3ff0f48, 0xc0012848a0})
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:71 +0x20f
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2af1a20, 0xc00184caa0}, {0x3ff16e0?, 0xc00200a0c0}, 0xc001284888, 0xc0000aa410)
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:121 +0x204
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1({0x2af1a20, 0xc00184caa0}, {0x3ff16e0, 0xc00200a0c0})
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:74 +0x187
go.etcd.io/etcd/etcdserver/api/v3rpc.newStreamInterceptor.func1({0x2af1a20, 0xc00184caa0}, {0x3ff16e0?, 0xc00200a0c0?}, 0x18?, 0xc0000aa410)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:225 +0x4b6
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2af1a20, 0xc00184caa0}, {0x3ff16e0, 0xc00200a0c0}, 0xc001284888, 0x3676508)
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:79 +0x352
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00109e780, {0x3ff4ad8, 0xc0003fb500}, 0xc000170500, 0xc00129ed50, 0x4c08860, 0x0)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1244 +0x182e
google.golang.org/grpc.(*Server).handleStream(0xc00109e780, {0x3ff4ad8, 0xc0003fb500}, 0xc000170500, 0x0)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1317 +0xf0a
google.golang.org/grpc.(*Server).serveStreams.func1.1()
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:722 +0xed
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:720 +0x170

goroutine 1483 [select]:
go.etcd.io/etcd/etcdserver.monitorFileDescriptor(0xc000309bc0, 0xc0017f38c0)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/metrics.go:215 +0x429
go.etcd.io/etcd/etcdserver.(*EtcdServer).Start.func3()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:733 +0x65
go.etcd.io/etcd/etcdserver.(*EtcdServer).goAttach.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2632 +0x7b
created by go.etcd.io/etcd/etcdserver.(*EtcdServer).goAttach
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2630 +0x2a6

goroutine 1487 [select]:
go.etcd.io/etcd/etcdserver/api/v3rpc.monitorLeader.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:247 +0x173
created by go.etcd.io/etcd/etcdserver/api/v3rpc.monitorLeader
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:242 +0x157

goroutine 1515 [chan receive, 9 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc000160840)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/clientconn.go:1164 +0x765
created by google.golang.org/grpc.(*addrConn).connect
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/clientconn.go:800 +0x105

goroutine 1504 [select, 9 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00184cc30, 0x1)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:395 +0x1ad
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0012d96e0)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:513 +0x11b
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:346 +0xf3
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:344 +0x26db

goroutine 1512 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x7fdf4f681e98, 0x72)
	/opt/hostedtoolcache/go/1.18.0/x64/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc00181c398, 0xc001019601?, 0x0)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_poll_runtime.go:83 +0xbd
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Accept(0xc00181c380)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_unix.go:614 +0x425
net.(*netFD).accept(0xc00181c380)
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/fd_unix.go:172 +0x4a
net.(*TCPListener).accept(0xc000307458)
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/tcpsock_posix.go:139 +0x45
net.(*TCPListener).Accept(0xc000307458)
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/tcpsock.go:288 +0x68
go.etcd.io/etcd/pkg/transport.(*limitListener).Accept(0xc000307488)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/transport/limit_listen.go:46 +0x6c
go.etcd.io/etcd/pkg/transport.(*keepaliveListener).Accept(0xc0013db790)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/transport/keepalive_listener.go:49 +0x45
github.com/soheilhy/cmux.(*cMux).Serve(0xc00184c9b0)
	/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:162 +0x122
go.etcd.io/etcd/embed.(*serveCtx).serve(0xc0004b8000, 0xc000390b00, 0xc0003906c8, {0x3fda3c0, 0xc001817680}, 0xc0012b24f0, {0xc001a020c0, 0x2, 0x2})
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/serve.go:201 +0x1c87
go.etcd.io/etcd/embed.(*Etcd).serveClients.func1(0x0?)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/etcd.go:751 +0x145
created by go.etcd.io/etcd/embed.(*Etcd).serveClients
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/etcd.go:750 +0xfcf

goroutine 1484 [select]:
go.etcd.io/etcd/etcdserver.(*EtcdServer).monitorVersions(0xc000390b00)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2456 +0x15f
go.etcd.io/etcd/etcdserver.(*EtcdServer).goAttach.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2632 +0x7b
created by go.etcd.io/etcd/etcdserver.(*EtcdServer).goAttach
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2630 +0x2a6

goroutine 1424 [select, 9 minutes]:
go.etcd.io/etcd/pkg/schedule.(*fifo).run(0xc0012d8d80)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/schedule/schedule.go:143 +0x409
created by go.etcd.io/etcd/pkg/schedule.NewFIFOScheduler
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/schedule/schedule.go:70 +0x2f7

goroutine 1496 [select, 9 minutes]:
go.etcd.io/etcd/pkg/schedule.(*fifo).run(0xc0012d9260)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/schedule/schedule.go:143 +0x409
created by go.etcd.io/etcd/pkg/schedule.NewFIFOScheduler
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/schedule/schedule.go:70 +0x2f7

goroutine 1500 [chan receive, 9 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
	/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:229
google.golang.org/grpc.(*Server).Serve(0xc00109e780, {0x3feabb0, 0xc00000e438})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:597 +0x4b7
go.etcd.io/etcd/embed.(*serveCtx).serve.func2()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/serve.go:118 +0x66
created by go.etcd.io/etcd/embed.(*serveCtx).serve
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/serve.go:118 +0x94b

goroutine 1491 [select]:
go.etcd.io/etcd/mvcc.(*watchableStore).syncWatchersLoop(0xc0000301e0)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/mvcc/watchable_store.go:227 +0x249
created by go.etcd.io/etcd/mvcc.newWatchableStore
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/mvcc/watchable_store.go:91 +0x9af

goroutine 1485 [select, 9 minutes]:
go.etcd.io/etcd/etcdserver.(*EtcdServer).linearizableReadLoop(0xc000390b00)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/v3_server.go:684 +0x2af
go.etcd.io/etcd/etcdserver.(*EtcdServer).goAttach.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2632 +0x7b
created by go.etcd.io/etcd/etcdserver.(*EtcdServer).goAttach
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:2630 +0x2a6

goroutine 1499 [select]:
go.etcd.io/etcd/etcdserver/api/v3rpc.monitorLeader.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:247 +0x173
created by go.etcd.io/etcd/etcdserver/api/v3rpc.monitorLeader
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:242 +0x157

goroutine 1509 [select]:
go.etcd.io/etcd/pkg/fileutil.purgeFile.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/fileutil/purge.go:80 +0x257
created by go.etcd.io/etcd/pkg/fileutil.purgeFile
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/fileutil/purge.go:34 +0x24f

goroutine 1526 [select, 9 minutes]:
github.com/tikv/pd/server/encryptionkm.(*KeyManager).StartBackgroundLoop(0xc0010bab80, {0x3febb18, 0xc0013a0080})
	/home/runner/work/pd/pd/server/encryptionkm/key_manager.go:228 +0xb05
created by github.com/tikv/pd/server/encryptionkm.(*testKeyManagerSuite).TestWatcher
	/home/runner/work/pd/pd/server/encryptionkm/key_manager_test.go:387 +0x4ab

goroutine 1489 [chan receive, 9 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
	/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:229
net/http.(*Server).Serve(0xc0000f0b60, {0x3feabb0, 0xc0000e62e8})
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/http/server.go:3039 +0x5a7
created by go.etcd.io/etcd/embed.(*Etcd).servePeers
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/etcd.go:548 +0xa2f

goroutine 1501 [chan receive, 9 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
	/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:229
net/http.(*Server).Serve(0xc00016cc40, {0x3feabb0, 0xc00000e468})
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/http/server.go:3039 +0x5a7
go.etcd.io/etcd/embed.(*serveCtx).serve.func3()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/serve.go:135 +0x59
created by go.etcd.io/etcd/embed.(*serveCtx).serve
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/serve.go:135 +0xdaf

goroutine 1421 [select]:
go.etcd.io/etcd/raft.(*node).run(0xc0012d87e0)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/raft/node.go:337 +0xc5d
created by go.etcd.io/etcd/raft.StartNode
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/raft/node.go:226 +0x31e

goroutine 1492 [select, 9 minutes]:
go.etcd.io/etcd/mvcc.(*watchableStore).syncVictimsLoop(0xc0000301e0)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/mvcc/watchable_store.go:253 +0x209
created by go.etcd.io/etcd/mvcc.newWatchableStore
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/mvcc/watchable_store.go:92 +0xa17

goroutine 1508 [select]:
go.etcd.io/etcd/pkg/fileutil.purgeFile.func1()
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/fileutil/purge.go:80 +0x257
created by go.etcd.io/etcd/pkg/fileutil.purgeFile
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/pkg/fileutil/purge.go:34 +0x24f

goroutine 1420 [select, 9 minutes]:
go.etcd.io/etcd/wal.(*filePipeline).run(0xc0013a1640)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/wal/file_pipeline.go:98 +0x1e5
created by go.etcd.io/etcd/wal.newFilePipeline
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/wal/file_pipeline.go:52 +0x277

goroutine 1417 [chan receive, 9 minutes]:
github.com/tikv/pd/server/encryptionkm.(*testKeyManagerSuite).TestWatcher(0x3600000000000000?, 0x16?)
	/home/runner/work/pd/pd/server/encryptionkm/key_manager_test.go:407 +0xa7c
reflect.Value.call({0x2cbef00?, 0x577e620?, 0x4213?}, {0x2d32be5, 0x4}, {0xc000e41eb8, 0x1, 0x1?})
	/opt/hostedtoolcache/go/1.18.0/x64/src/reflect/value.go:556 +0xd9d
reflect.Value.Call({0x2cbef00?, 0x577e620?, 0xc000047620?}, {0xc0000e0eb8, 0x1, 0x1})
	/opt/hostedtoolcache/go/1.18.0/x64/src/reflect/value.go:339 +0xd8
github.com/pingcap/check.(*suiteRunner).forkTest.func1(0xc00204d2c0)
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/check.go:850 +0xa72
github.com/pingcap/check.(*suiteRunner).forkCall.func1()
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/check.go:739 +0x11f
created by github.com/pingcap/check.(*suiteRunner).forkCall
	/home/runner/go/pkg/mod/github.com/pingcap/[email protected]/check.go:734 +0x5de

goroutine 1419 [select]:
go.etcd.io/etcd/mvcc/backend.(*backend).run(0xc00181c480)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/mvcc/backend/backend.go:323 +0x22f
created by go.etcd.io/etcd/mvcc/backend.newBackend
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/mvcc/backend/backend.go:185 +0x965

goroutine 1503 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x7fdf4f681cb8, 0x72)
	/opt/hostedtoolcache/go/1.18.0/x64/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc00181ca98, 0xc001184000?, 0x0)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_poll_runtime.go:83 +0xbd
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc00181ca80, {0xc001184000, 0x8000, 0x8000})
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_unix.go:167 +0x415
net.(*netFD).Read(0xc00181ca80, {0xc001184000, 0x8000, 0x8000})
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/fd_posix.go:55 +0x51
net.(*conn).Read(0xc000011070, {0xc001184000, 0x8000, 0x8000})
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/net.go:183 +0xb1
bufio.(*Reader).Read(0xc0012d9620, {0xc00016cd58, 0x9, 0x9})
	/opt/hostedtoolcache/go/1.18.0/x64/src/bufio/bufio.go:236 +0x4db
io.ReadAtLeast({0x3fd7320, 0xc0012d9620}, {0xc00016cd58, 0x9, 0x9}, 0x9)
	/opt/hostedtoolcache/go/1.18.0/x64/src/io/io.go:331 +0xde
io.ReadFull(...)
	/opt/hostedtoolcache/go/1.18.0/x64/src/io/io.go:350
golang.org/x/net/http2.readFrameHeader({0xc00016cd58?, 0x9?, 0x9?}, {0x3fd7320?, 0xc0012d9620?})
	/home/runner/go/pkg/mod/golang.org/x/[email protected][803](https://github.com/tikv/pd/runs/6207105310?check_suite_focus=true#step:5:803)19-a5a99cb37ef4/http2/frame.go:237 +0x96
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00016cd20)
	/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:492 +0x108
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00016ea80)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:1264 +0x345
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:300 +0x20d2

goroutine 1495 [select]:
go.etcd.io/etcd/etcdserver.(*EtcdServer).run(0xc000390b00)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:1036 +0xc2f
created by go.etcd.io/etcd/etcdserver.(*EtcdServer).start
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/server.go:[805](https://github.com/tikv/pd/runs/6207105310?check_suite_focus=true#step:5:805) +0x1792

goroutine 1536 [select, 9 minutes]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0000aa230, {0xc001284778, 0x5, 0x5})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:186 +0x13e
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000aa230, {0xc001284778, 0x5, 0x5})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:166 +0x365
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012f67e0, {0xc001284778, 0x5, 0x5})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:479 +0x66
io.ReadAtLeast({0x3fd9ec0, 0xc0012f67e0}, {0xc001284778, 0x5, 0x5}, 0x5)
	/opt/hostedtoolcache/go/1.18.0/x64/src/io/io.go:331 +0xde
io.ReadFull(...)
	/opt/hostedtoolcache/go/1.18.0/x64/src/io/io.go:350
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000170400, {0xc001284778, 0x5, 0x5})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:463 +0x125
google.golang.org/grpc.(*parser).recvMsg(0xc001284768, 0x7fffffff)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:508 +0x75
google.golang.org/grpc.recvAndDecompress(0x577e620?, 0xc000170400, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:639 +0x88
google.golang.org/grpc.recv(0x0?, {0x7fdf245ea3a8, 0x577e620}, 0xf?, {0x0, 0x0}, {0x2cb2640, 0xc00184d[810](https://github.com/tikv/pd/runs/6207105310?check_suite_focus=true#step:5:810)}, 0xc00116b1a0?, 0x0, ...)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:707 +0xba
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0010bb580, {0x2cb2640?, 0xc00184d810}, 0x0)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:885 +0x505
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00138397c?)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:736 +0x57
google.golang.org/grpc.(*clientStream).withRetry(0xc0013838c0, 0xc000e48e38, 0xc000e48e08)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:590 +0x323
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0013838c0, {0x2cb2640, 0xc00184d810})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:735 +0x186
go.etcd.io/etcd/etcdserver/etcdserverpb.(*watchWatchClient).Recv(0xc0012b3950)
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/etcdserverpb/rpc.pb.go:3702 +0x68
go.etcd.io/etcd/clientv3.(*watchGrpcStream).serveWatchClient(0xc001158000, {0x3ff3328, 0xc0012b3950})
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/clientv3/watch.go:716 +0x75
created by go.etcd.io/etcd/clientv3.(*watchGrpcStream).newWatchClient
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/clientv3/watch.go:871 +0x9ca

goroutine 1488 [chan receive, 9 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
	/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:229
google.golang.org/grpc.(*Server).Serve(0xc001861980, {0x3feabb0, 0xc0000e62d0})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:597 +0x4b7
created by go.etcd.io/etcd/embed.(*Etcd).servePeers
	/home/runner/go/pkg/mod/go.etcd.io/[email protected]/embed/etcd.go:542 +0x63b

goroutine 1520 [select, 9 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0003fb500)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:968 +0x3cf
created by google.golang.org/grpc/internal/transport.newHTTP2Server
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:302 +0x2329

goroutine 1521 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x7fdf4f681da8, 0x72)
	/opt/hostedtoolcache/go/1.18.0/x64/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc0010ba898, 0xc001270000?, 0x0)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_poll_runtime.go:83 +0xbd
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc0010ba880, {0xc001270000, 0x8000, 0x8000})
	/opt/hostedtoolcache/go/1.18.0/x64/src/internal/poll/fd_unix.go:167 +0x415
net.(*netFD).Read(0xc0010ba880, {0xc001270000, 0x8000, 0x8000})
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/fd_posix.go:55 +0x51
net.(*conn).Read(0xc000126c70, {0xc001270000, 0x8000, 0x8000})
	/opt/hostedtoolcache/go/1.18.0/x64/src/net/net.go:183 +0xb1
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc0004b2a20, {0xc001270000, 0x8000, 0x8000})
	/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/buffer.go:53 +0x2f8
github.com/soheilhy/cmux.(*MuxConn).Read(0xc0004b2a10, {0xc001270000, 0x8000, 0x8000})
	/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:260 +0x51
bufio.(*Reader).Read(0xc00130fc80, {0xc0000f0e38, 0x9, 0x9})
	/opt/hostedtoolcache/go/1.18.0/x64/src/bufio/bufio.go:236 +0x4db
io.ReadAtLeast({0x3fd7320, 0xc00130fc80}, {0xc0000f0e38, 0x9, 0x9}, 0x9)
	/opt/hostedtoolcache/go/1.18.0/x64/src/io/io.go:331 +0xde
io.ReadFull(...)
	/opt/hostedtoolcache/go/1.18.0/x64/src/io/io.go:350
golang.org/x/net/http2.readFrameHeader({0xc0000f0e38?, 0x9?, 0x9?}, {0x3fd7320?, 0xc00130fc80?})
	/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x96
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0000f0e00)
	/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:492 +0x108
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0003fb500, 0xc0017e2[930](https://github.com/tikv/pd/runs/6207105310?check_suite_focus=true#step:5:930)?, 0x0?)
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:458 +0x145
google.golang.org/grpc.(*Server).serveStreams(0xc00109e780, {0x3ff4ad8?, 0xc0003fb500})
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:718 +0x1d9
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:679 +0x4d
created by google.golang.org/grpc.(*Server).handleRawConn
	/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:678 +0x5e5
@JmPotato JmPotato added the type/ci The issue is related to CI. label Apr 29, 2022
@rleungx rleungx mentioned this issue Jul 12, 2023
Closed
@lhy1024
Copy link
Contributor

lhy1024 commented Aug 1, 2023

https://github.com/tikv/pd/actions/runs/5724040701/job/15509738226?pr=6856

@lhy1024 lhy1024 mentioned this issue Aug 21, 2024
Open
@rleungx rleungx closed this as completed Sep 6, 2024
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
type/ci The issue is related to CI.
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
3 participants
@JmPotato @lhy1024 @rleungx