store: Flaky(?) too long running unit tests

[bwplotka]

Something is wrong with FAIL github.com/improbable-eng/thanos/pkg/store 600.006s tests... Those timeouts a lot on CI: https://circleci.com/gh/improbable-eng/thanos/1865?utm_campaign=vcs-integration-link&utm_medium=referral&utm_source=github-build-link

Investigating.

[bwplotka]

Well at least I can repro it!

SIGQUIT: quit
PC=0x45d461 m=0 sigcode=0

goroutine 0 [idle]:
runtime.futex(0x12fee68, 0x0, 0x0, 0x0, 0x0, 0xc42019e800, 0x0, 0x0, 0x7ffe99c1bca8, 0x410fcb, ...)
	/usr/local/go/src/runtime/sys_linux_amd64.s:526 +0x21
runtime.futexsleep(0x12fee68, 0xc400000000, 0xffffffffffffffff)
	/usr/local/go/src/runtime/os_linux.go:45 +0x4b
runtime.notesleep(0x12fee68)
	/usr/local/go/src/runtime/lock_futex.go:151 +0x9b
runtime.stopm()
	/usr/local/go/src/runtime/proc.go:1947 +0xe5
runtime.findrunnable(0xc420060500, 0x0)
	/usr/local/go/src/runtime/proc.go:2410 +0x50c
runtime.schedule()
	/usr/local/go/src/runtime/proc.go:2536 +0x13b
runtime.park_m(0xc420492000)
	/usr/local/go/src/runtime/proc.go:2599 +0xb6
runtime.mcall(0x0)
	/usr/local/go/src/runtime/asm_amd64.s:351 +0x5b

goroutine 1 [chan receive, 9 minutes]:
testing.(*T).Run(0xc420f603c0, 0xda6617, 0x33, 0xdb6f68, 0x47fc01)
	/usr/local/go/src/testing/testing.go:825 +0x301
testing.runTests.func1(0xc4202000f0)
	/usr/local/go/src/testing/testing.go:1063 +0x64
testing.tRunner(0xc4202000f0, 0xc420287df8)
	/usr/local/go/src/testing/testing.go:777 +0xd0
testing.runTests(0xc420260180, 0x12f2780, 0x12, 0x12, 0x411fe9)
	/usr/local/go/src/testing/testing.go:1061 +0x2c4
testing.(*M).Run(0xc420108600, 0x0)
	/usr/local/go/src/testing/testing.go:978 +0x171
main.main()
	_testmain.go:76 +0x151

goroutine 22 [select]:
github.com/improbable-eng/thanos/vendor/go.opencensus.io/stats/view.(*worker).start(0xc4200e0b40)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/vendor/go.opencensus.io/stats/view/worker.go:146 +0x11d
created by github.com/improbable-eng/thanos/vendor/go.opencensus.io/stats/view.init.0
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/vendor/go.opencensus.io/stats/view/worker.go:29 +0x58

goroutine 1046 [chan send, 9 minutes]:
github.com/improbable-eng/thanos/pkg/store.(*ProxyStore).Series(0xc42303eba0, 0xc4204b8f28, 0xe32120, 0xc4204bd0e0, 0x1, 0x1)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:132 +0xac1
github.com/improbable-eng/thanos/pkg/store.TestProxyStore_Series_RegressionFillResponseChannel(0xc420f603c0)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy_test.go:503 +0xa24
testing.tRunner(0xc420f603c0, 0xdb6f68)
	/usr/local/go/src/testing/testing.go:777 +0xd0
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:824 +0x2e0

goroutine 1054 [chan send, 9 minutes]:
github.com/improbable-eng/thanos/pkg/store.(*streamSeriesSet).fetchLoop(0xc42303eea0)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:211 +0x15f
created by github.com/improbable-eng/thanos/pkg/store.startStreamSeriesSet
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:194 +0xb5

goroutine 1048 [chan send, 9 minutes]:
github.com/improbable-eng/thanos/pkg/store.(*streamSeriesSet).fetchLoop(0xc42303ec60)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:211 +0x15f
created by github.com/improbable-eng/thanos/pkg/store.startStreamSeriesSet
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:194 +0xb5

goroutine 1053 [chan send, 9 minutes]:
github.com/improbable-eng/thanos/pkg/store.(*streamSeriesSet).fetchLoop(0xc42303ee40)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:211 +0x15f
created by github.com/improbable-eng/thanos/pkg/store.startStreamSeriesSet
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:194 +0xb5

goroutine 1050 [chan send, 9 minutes]:
github.com/improbable-eng/thanos/pkg/store.(*streamSeriesSet).fetchLoop(0xc42303ed20)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:211 +0x15f
created by github.com/improbable-eng/thanos/pkg/store.startStreamSeriesSet
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:194 +0xb5

goroutine 1052 [chan send, 9 minutes]:
github.com/improbable-eng/thanos/pkg/store.(*streamSeriesSet).fetchLoop(0xc42303ede0)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:211 +0x15f
created by github.com/improbable-eng/thanos/pkg/store.startStreamSeriesSet
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:194 +0xb5

goroutine 1049 [chan send, 9 minutes]:
github.com/improbable-eng/thanos/pkg/store.(*streamSeriesSet).fetchLoop(0xc42303ecc0)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:211 +0x15f
created by github.com/improbable-eng/thanos/pkg/store.startStreamSeriesSet
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:194 +0xb5

goroutine 1051 [chan send, 9 minutes]:
github.com/improbable-eng/thanos/pkg/store.(*streamSeriesSet).fetchLoop(0xc42303ed80)
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:211 +0x15f
created by github.com/improbable-eng/thanos/pkg/store.startStreamSeriesSet
	/home/bartek/Repos/thanosGo/src/github.com/improbable-eng/thanos/pkg/store/proxy.go:194 +0xb5

rax    0xca
rbx    0x12fed20
rcx    0x45d463
rdx    0x0
rdi    0x12fee68
rsi    0x0
rbp    0x7ffe99c1bc70
rsp    0x7ffe99c1bc28
r8     0x0
r9     0x0
r10    0x0
r11    0x286
r12    0x3f6355
r13    0x0
r14    0x0
r15    0x45ac30
rip    0x45d461
rflags 0x286
cs     0x33
fs     0x0
gs     0x0
*** Test killed with quit: ran too long (10m0s).
FAIL	github.com/improbable-eng/thanos/pkg/store	600.016s
?   	github.com/improbable-eng/thanos/pkg/store/prompb	[no test files]
?   	github.com/improbable-eng/thanos/pkg/store/storepb	[no test files]