runtime: "morestack on g0" on freebsd-386

[bcmills]

2021-04-30T18:14:41-d9bfda8/freebsd-386-12_2

##### ../misc/cgo/errors
--- FAIL: TestReportsTypeErrors (0.50s)
    --- FAIL: TestReportsTypeErrors/issue33061.go (3.60s)
        errors_test.go:82: expected error output to contain `issue33061\.go:16:`
        errors_test.go:87: actual output:
            fatal: morestack on g0
            SIGTRAP: trace trap
            PC=0x80a6752 m=0 sigcode=1
            
            goroutine 0 [idle]:
            runtime.abort()
            	/tmp/workdir/go/src/runtime/asm_386.s:843 +0x2
            runtime.morestack()
            	/tmp/workdir/go/src/runtime/asm_386.s:418 +0x1f
            
            goroutine 22 [running]:
FAIL

This may be related to #31683.

CC @ianlancetaylor @cherrymui @prattmic @mknyszek

[bcmills]

2021-06-15T10:13:08-abc56fd/freebsd-386-12_2 has a more complete trace, this time during go test on the builder:

fatal: morestack on g0
SIGTRAP: trace trap
PC=0x80a6b82 m=7 sigcode=1

goroutine 4 [running]:
runtime.abort()
	/tmp/workdir/go/src/runtime/asm_386.s:847 +0x2 fp=0x39433f70 sp=0x39433f6c pc=0x80a6b82
runtime.morestack()
	/tmp/workdir/go/src/runtime/asm_386.s:422 +0x1f fp=0x39433f74 sp=0x39433f70 pc=0x80a594f
created by runtime.gcenable
	/tmp/workdir/go/src/runtime/mgc.go:182 +0x78

goroutine 1 [semacquire]:
runtime.gopark(0x859ef38, 0x88bd580, 0x12, 0x19, 0x4)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x39442bac sp=0x39442b98 pc=0x807d686
runtime.goparkunlock(...)
	/tmp/workdir/go/src/runtime/proc.go:372
runtime.semacquire1(0x39897908, 0x0, 0x1, 0x0)
	/tmp/workdir/go/src/runtime/sema.go:144 +0x22b fp=0x39442be0 sp=0x39442bac pc=0x808d4eb
sync.runtime_Semacquire(0x39897908)
	/tmp/workdir/go/src/runtime/sema.go:56 +0x30 fp=0x39442bf4 sp=0x39442be0 pc=0x80a4660
sync.(*WaitGroup).Wait(0x39897900)
	/tmp/workdir/go/src/sync/waitgroup.go:130 +0x81 fp=0x39442c18 sp=0x39442bf4 pc=0x80afd61
cmd/go/internal/work.(*Builder).Do(0x394b8ba0, {0x862bf80, 0x3941c108}, 0x39993080)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:208 +0x3d9 fp=0x39442ca8 sp=0x39442c18 pc=0x8435e09
cmd/go/internal/test.runTest({0x862bf80, 0x3941c108}, 0x88b1e80, {0x39416060, 0x7, 0x8})
	/tmp/workdir/go/src/cmd/go/internal/test/test.go:826 +0x2000 fp=0x39442ef8 sp=0x39442ca8 pc=0x8498e40
main.main()
	/tmp/workdir/go/src/cmd/go/main.go:201 +0xb28 fp=0x39442fc8 sp=0x39442ef8 pc=0x84b02a8
runtime.main()
	/tmp/workdir/go/src/runtime/proc.go:255 +0x238 fp=0x39442ff0 sp=0x39442fc8 pc=0x807d2a8
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x39442ff4 sp=0x39442ff0 pc=0x80a6d11

goroutine 2 [force gc (idle)]:
runtime.gopark(0x859ef38, 0x88b6fc8, 0x11, 0x14, 0x1)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x39432fdc sp=0x39432fc8 pc=0x807d686
runtime.goparkunlock(...)
	/tmp/workdir/go/src/runtime/proc.go:372
runtime.forcegchelper()
	/tmp/workdir/go/src/runtime/proc.go:306 +0xb8 fp=0x39432ff0 sp=0x39432fdc pc=0x807d518
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x39432ff4 sp=0x39432ff0 pc=0x80a6d11
created by runtime.init.6
	/tmp/workdir/go/src/runtime/proc.go:294 +0x25

goroutine 3 [GC sweep wait]:
runtime.gopark(0x859ef38, 0x88b7660, 0xc, 0x14, 0x1)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x394337dc sp=0x394337c8 pc=0x807d686
runtime.goparkunlock(...)
	/tmp/workdir/go/src/runtime/proc.go:372
runtime.bgsweep()
	/tmp/workdir/go/src/runtime/mgcsweep.go:182 +0xdc fp=0x394337f0 sp=0x394337dc pc=0x80699cc
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x394337f4 sp=0x394337f0 pc=0x80a6d11
created by runtime.gcenable
	/tmp/workdir/go/src/runtime/mgc.go:181 +0x62

goroutine 5 [finalizer wait]:
runtime.gopark(0x859ef38, 0x88cc0c8, 0x10, 0x14, 0x1)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x39432794 sp=0x39432780 pc=0x807d686
runtime.goparkunlock(...)
	/tmp/workdir/go/src/runtime/proc.go:372
runtime.runfinq()
	/tmp/workdir/go/src/runtime/mfinal.go:177 +0x9a fp=0x394327f0 sp=0x39432794 pc=0x805e1da
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x394327f4 sp=0x394327f0 pc=0x80a6d11
created by runtime.createfing
	/tmp/workdir/go/src/runtime/mfinal.go:157 +0x56

goroutine 10 [GC worker (idle)]:
runtime.gopark(0x859ee8c, 0x3941e4f8, 0x18, 0x14, 0x0)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x39435fa0 sp=0x39435f8c pc=0x807d686
runtime.gcBgMarkWorker()
	/tmp/workdir/go/src/runtime/mgc.go:1200 +0xf1 fp=0x39435ff0 sp=0x39435fa0 pc=0x8060f51
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x39435ff4 sp=0x39435ff0 pc=0x80a6d11
created by runtime.gcBgMarkStartWorkers
	/tmp/workdir/go/src/runtime/mgc.go:1124 +0x27

goroutine 18 [GC worker (idle)]:
runtime.gopark(0x859ee8c, 0x396301e0, 0x18, 0x14, 0x0)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x39434fa0 sp=0x39434f8c pc=0x807d686
runtime.gcBgMarkWorker()
	/tmp/workdir/go/src/runtime/mgc.go:1200 +0xf1 fp=0x39434ff0 sp=0x39434fa0 pc=0x8060f51
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x39434ff4 sp=0x39434ff0 pc=0x80a6d11
created by runtime.gcBgMarkStartWorkers
	/tmp/workdir/go/src/runtime/mgc.go:1124 +0x27

goroutine 11 [GC worker (idle)]:
runtime.gopark(0x859ee8c, 0x396301f8, 0x18, 0x14, 0x0)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x3942e7a0 sp=0x3942e78c pc=0x807d686
runtime.gcBgMarkWorker()
	/tmp/workdir/go/src/runtime/mgc.go:1200 +0xf1 fp=0x3942e7f0 sp=0x3942e7a0 pc=0x8060f51
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x3942e7f4 sp=0x3942e7f0 pc=0x80a6d11
created by runtime.gcBgMarkStartWorkers
	/tmp/workdir/go/src/runtime/mgc.go:1124 +0x27

goroutine 19 [GC worker (idle)]:
runtime.gopark(0x859ee8c, 0x395ac390, 0x18, 0x14, 0x0)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x397e87a0 sp=0x397e878c pc=0x807d686
runtime.gcBgMarkWorker()
	/tmp/workdir/go/src/runtime/mgc.go:1200 +0xf1 fp=0x397e87f0 sp=0x397e87a0 pc=0x8060f51
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x397e87f4 sp=0x397e87f0 pc=0x80a6d11
created by runtime.gcBgMarkStartWorkers
	/tmp/workdir/go/src/runtime/mgc.go:1124 +0x27

goroutine 354 [runnable]:
syscall.Syscall(0xb, 0x0, 0x0, 0x0)
	/tmp/workdir/go/src/syscall/asm_unix_386.s:20 +0x5 fp=0x394d9ac4 sp=0x394d9ac0 pc=0x80b8745
syscall.Close(0xb)
	/tmp/workdir/go/src/syscall/zsyscall_freebsd_386.go:376 +0x37 fp=0x394d9ae4 sp=0x394d9ac4 pc=0x80b6187
syscall.forkExec({0x39641500, 0x32}, {0x3978e000, 0x5, 0x6}, 0x394d9bd4)
	/tmp/workdir/go/src/syscall/exec_unix.go:227 +0x56b fp=0x394d9b8c sp=0x394d9ae4 pc=0x80b1adb
syscall.StartProcess(...)
	/tmp/workdir/go/src/syscall/exec_unix.go:264
os.startProcess({0x39641500, 0x32}, {0x3978e000, 0x5, 0x6}, 0x394d9c98)
	/tmp/workdir/go/src/os/exec_posix.go:55 +0x256 fp=0x394d9bfc sp=0x394d9b8c pc=0x8108576
os.StartProcess({0x39641500, 0x32}, {0x3978e000, 0x5, 0x6}, 0x394d9c98)
	/tmp/workdir/go/src/os/exec.go:109 +0x57 fp=0x394d9c24 sp=0x394d9bfc pc=0x8108017
os/exec.(*Cmd).Start(0x39642790)
	/tmp/workdir/go/src/os/exec/exec.go:422 +0x588 fp=0x394d9cc0 sp=0x394d9c24 pc=0x8134e28
cmd/go/internal/test.(*runCache).builderRunTest(0x397db130, 0x394b8ba0, {0x862bf80, 0x3941c108}, 0x395e7c30)
	/tmp/workdir/go/src/cmd/go/internal/test/test.go:1230 +0xaa8 fp=0x394d9e90 sp=0x394d9cc0 pc=0x849b998
cmd/go/internal/test.(*runCache).builderRunTest-fm(0x394b8ba0, {0x862bf80, 0x3941c108}, 0x395e7c30)
	/tmp/workdir/go/src/cmd/go/internal/test/test.go:1108 +0x41 fp=0x394d9eb0 sp=0x394d9e90 pc=0x84a3ec1
cmd/go/internal/work.(*Builder).Do.func2({0x862bf80, 0x3941c108}, 0x395e7c30)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:137 +0x539 fp=0x394d9f88 sp=0x394d9eb0 pc=0x8436649
cmd/go/internal/work.(*Builder).Do.func3(0x394930f8, 0x39897900, 0x394b8ba0, 0x398f2740)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:199 +0x92 fp=0x394d9fe0 sp=0x394d9f88 pc=0x8435fe2
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x394d9fe4 sp=0x394d9fe0 pc=0x80a6d11
created by cmd/go/internal/work.(*Builder).Do
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:185 +0x3be

goroutine 368 [IO wait]:
runtime.gopark(0x859ef24, 0x39063330, 0x2, 0x1b, 0x5)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x397ebe4c sp=0x397ebe38 pc=0x807d686
runtime.netpollblock(0x39063318, 0x72, 0x0)
	/tmp/workdir/go/src/runtime/netpoll.go:445 +0xf9 fp=0x397ebe64 sp=0x397ebe4c pc=0x8076009
internal/poll.runtime_pollWait(0x39063318, 0x72)
	/tmp/workdir/go/src/runtime/netpoll.go:229 +0x88 fp=0x397ebe78 sp=0x397ebe64 pc=0x80a3188
internal/poll.(*pollDesc).wait(0x3984c2d4, 0x72, 0x1)
	/tmp/workdir/go/src/internal/poll/fd_poll_runtime.go:84 +0x31 fp=0x397ebe8c sp=0x397ebe78 pc=0x8101ad1
internal/poll.(*pollDesc).waitRead(...)
	/tmp/workdir/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0x3984c2c0, {0x396ca400, 0x200, 0x200})
	/tmp/workdir/go/src/internal/poll/fd_unix.go:167 +0x1ce fp=0x397ebec4 sp=0x397ebe8c pc=0x810277e
os.(*File).read(...)
	/tmp/workdir/go/src/os/file_posix.go:32
os.(*File).Read(0x3958d570, {0x396ca400, 0x200, 0x200})
	/tmp/workdir/go/src/os/file.go:119 +0x5c fp=0x397ebf00 sp=0x397ebec4 pc=0x81091ec
bytes.(*Buffer).ReadFrom(0x394a2630, {0x8624db0, 0x3958d570})
	/tmp/workdir/go/src/bytes/buffer.go:204 +0x8f fp=0x397ebf44 sp=0x397ebf00 pc=0x812fd4f
io.copyBuffer({0x862448c, 0x394a2630}, {0x8624db0, 0x3958d570}, {0x0, 0x0, 0x0})
	/tmp/workdir/go/src/io/io.go:409 +0x125 fp=0x397ebf98 sp=0x397ebf44 pc=0x80fd485
io.Copy(...)
	/tmp/workdir/go/src/io/io.go:382
os/exec.(*Cmd).writerDescriptor.func1()
	/tmp/workdir/go/src/os/exec/exec.go:311 +0x4f fp=0x397ebfd4 sp=0x397ebf98 pc=0x81347af
os/exec.(*Cmd).Start.func1(0x39822bb0, 0x3965e0e0)
	/tmp/workdir/go/src/os/exec/exec.go:441 +0x17 fp=0x397ebfe8 sp=0x397ebfd4 pc=0x8135157
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x397ebfec sp=0x397ebfe8 pc=0x80a6d11
created by os/exec.(*Cmd).Start
	/tmp/workdir/go/src/os/exec/exec.go:440 +0x791

goroutine 336 [syscall]:
syscall.Syscall9(0x0, 0xbb5, 0x0, 0x0, 0x18, 0x0, 0x0, 0x0, 0x0, 0x0)
	/tmp/workdir/go/src/syscall/asm_unix_386.s:71 +0x5 fp=0x39967ad0 sp=0x39967acc pc=0x80b87e5
os.(*Process).blockUntilWaitable(0x394ee600)
	/tmp/workdir/go/src/os/wait_wait6.go:27 +0x69 fp=0x39967b0c sp=0x39967ad0 pc=0x810fc39
os.(*Process).wait(0x394ee600)
	/tmp/workdir/go/src/os/exec_unix.go:23 +0x28 fp=0x39967b44 sp=0x39967b0c pc=0x81089b8
os.(*Process).Wait(...)
	/tmp/workdir/go/src/os/exec.go:132
os/exec.(*Cmd).Wait(0x39822bb0)
	/tmp/workdir/go/src/os/exec/exec.go:507 +0x4d fp=0x39967b80 sp=0x39967b44 pc=0x813521d
os/exec.(*Cmd).Run(0x39822bb0)
	/tmp/workdir/go/src/os/exec/exec.go:341 +0x43 fp=0x39967b90 sp=0x39967b80 pc=0x8134883
cmd/go/internal/work.(*Builder).runOut(0x394b8ba0, 0x399922c0, {0x855409b, 0x1}, {0x39967d38, 0x0, 0x0}, {0x39967dcc, 0x8, 0x8})
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:2096 +0x729 fp=0x39967c9c sp=0x39967b90 pc=0x8447b79
cmd/go/internal/work.(*Builder).run(0x394b8ba0, 0x399922c0, {0x855409b, 0x1}, {0x395ac1b0, 0x12}, {0x39967d38, 0x0, 0x0}, {0x39967dcc, ...})
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:2008 +0x76 fp=0x39967d00 sp=0x39967c9c pc=0x8447136
cmd/go/internal/work.gcToolchain.ld({}, 0x394b8ba0, 0x399922c0, {0x395e5280, 0x34}, {0x39844fc0, 0x37}, {0x39571230, 0x30})
	/tmp/workdir/go/src/cmd/go/internal/work/gc.go:672 +0xce5 fp=0x39967e10 sp=0x39967d00 pc=0x8459be5
cmd/go/internal/work.(*gcToolchain).ld(0x88cc154, 0x394b8ba0, 0x399922c0, {0x395e5280, 0x34}, {0x39844fc0, 0x37}, {0x39571230, 0x30})
	<autogenerated>:1 +0x62 fp=0x39967e3c sp=0x39967e10 pc=0x8467ea2
cmd/go/internal/work.(*Builder).link(0x394b8ba0, {0x862bf80, 0x3941c108}, 0x399922c0)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:1328 +0x2f4 fp=0x39967eb0 sp=0x39967e3c pc=0x8442014
cmd/go/internal/work.(*Builder).Do.func2({0x862bf80, 0x3941c108}, 0x399922c0)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:137 +0x539 fp=0x39967f88 sp=0x39967eb0 pc=0x8436649
cmd/go/internal/work.(*Builder).Do.func3(0x394930f8, 0x39897900, 0x394b8ba0, 0x398f2740)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:199 +0x92 fp=0x39967fe0 sp=0x39967f88 pc=0x8435fe2
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x39967fe4 sp=0x39967fe0 pc=0x80a6d11
created by cmd/go/internal/work.(*Builder).Do
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:185 +0x3be

goroutine 337 [select]:
runtime.gopark(0x859ef60, 0x0, 0x9, 0x18, 0x1)
	/tmp/workdir/go/src/runtime/proc.go:366 +0xf6 fp=0x39963e94 sp=0x39963e80 pc=0x807d686
runtime.selectgo(0x39963fc4, 0x39963fac, 0x0, 0x0, 0x2, 0x1)
	/tmp/workdir/go/src/runtime/select.go:327 +0xbd6 fp=0x39963f88 sp=0x39963e94 pc=0x808c946
cmd/go/internal/work.(*Builder).Do.func3(0x394930f8, 0x39897900, 0x394b8ba0, 0x398f2740)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:189 +0xf8 fp=0x39963fe0 sp=0x39963f88 pc=0x8436048
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x39963fe4 sp=0x39963fe0 pc=0x80a6d11
created by cmd/go/internal/work.(*Builder).Do
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:185 +0x3be

goroutine 335 [runnable]:
crypto/sha256.(*digest).Write(0x39445b98, {0x39666000, 0x7c00, 0x7c80})
	/tmp/workdir/go/src/crypto/sha256/sha256.go:184 +0x1ec fp=0x39445af8 sp=0x39445af4 pc=0x81d281c
cmd/internal/buildid.FindAndHash({0x8624db0, 0x398a6048}, {0x39660000, 0x53}, 0x0)
	/tmp/workdir/go/src/cmd/internal/buildid/rewrite.go:77 +0x3f2 fp=0x39445c38 sp=0x39445af8 pc=0x8427412
cmd/go/internal/work.(*Builder).updateBuildID(0x394b8ba0, 0x39992b00, {0x395e4a80, 0x34}, 0x0)
	/tmp/workdir/go/src/cmd/go/internal/work/buildid.go:628 +0x370 fp=0x39445e3c sp=0x39445c38 pc=0x8434ef0
cmd/go/internal/work.(*Builder).link(0x394b8ba0, {0x862bf80, 0x3941c108}, 0x39992b00)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:1348 +0x33e fp=0x39445eb0 sp=0x39445e3c pc=0x844205e
cmd/go/internal/work.(*Builder).Do.func2({0x862bf80, 0x3941c108}, 0x39992b00)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:137 +0x539 fp=0x39445f88 sp=0x39445eb0 pc=0x8436649
cmd/go/internal/work.(*Builder).Do.func3(0x394930f8, 0x39897900, 0x394b8ba0, 0x398f2740)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:199 +0x92 fp=0x39445fe0 sp=0x39445f88 pc=0x8435fe2
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_386.s:1319 +0x1 fp=0x39445fe4 sp=0x39445fe0 pc=0x80a6d11
created by cmd/go/internal/work.(*Builder).Do
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:185 +0x3be

eax    0x17
ebx    0x39604000
ecx    0x17
edx    0x0
edi    0x3958d150
esi    0x396060f0
ebp    0xcb6b8e9
esp    0x39433f6c
eip    0x80a6b82
eflags 0x200206
cs     0x33
fs     0x13
gs     0x1b
go tool dist: Failed: exit status 2

[bcmills]

I wonder if this is connected to #34988, given that they both involve apparent memory corruption around the time of starting a subprocess on BSD variants.

[toothrot]

A freebsd-amd64 example: https://storage.googleapis.com/go-build-log/7b8706d1/freebsd-amd64-12_2_a6a4aad9.log

[zhangfannie]

I also encountered this issue on freebsd-amd64. Please see https://storage.googleapis.com/go-build-log/2c4e7531/freebsd-amd64-12_2_fcb2c644.log.
Just curious, is this a random issue? Thank you.