overlayfs panics in runsc

[hbhasker]

Repro,

Start runsc with --overlay.

Copy large amounts of data (I was copying over around 148 Gigs).

Ctrl-c midway of copy

ls -la (panics)

panic: an overlay cannot revalidate file objects from the lower fs

goroutine 16529 [running]:
google3/third_party/golang/gvisor/pkg/sentry/fs/fs.(*overlayMountSourceOperations).Revalidate(0xc00058d350, 0xf01c60, 0xc000207500, 0xc00193b000, 0x7, 0xc000726900, 0xc000566300, 0xc00157b8c8)
        third_party/golang/gvisor/pkg/sentry/fs/mount_overlay.go:66 +0x176
google3/third_party/golang/gvisor/pkg/sentry/fs/fs.(*Dirent).walk(0xc000724750, 0xf01c60, 0xc000207500, 0xc00015a750, 0xc00193b000, 0x7, 0xc00157b901, 0x701150, 0xc000726900, 0xf01c60)
        third_party/golang/gvisor/pkg/sentry/fs/dirent.go:505 +0x71e
google3/third_party/golang/gvisor/pkg/sentry/fs/fs.(*Dirent).Walk(0xc000724750, 0xf01c60, 0xc000207500, 0xc00015a750, 0xc00193b000, 0x7, 0xc00157bb10, 0x660765, 0xc001868000)
        third_party/golang/gvisor/pkg/sentry/fs/dirent.go:616 +0xd3
google3/third_party/golang/gvisor/pkg/sentry/fs/fs.(*MountNamespace).FindLink(0xc0001620f0, 0xf01c60, 0xc000207500, 0xc00015a750, 0xc000724750, 0xc00193b000, 0x7, 0xc00157bb10, 0xc0003fa630, 0x0, ...)
        third_party/golang/gvisor/pkg/sentry/fs/mounts.go:405 +0x2d8
google3/third_party/golang/gvisor/pkg/sentry/syscalls/linux/linux.fileOpOn(0xc000207500, 0x7fc1ffffff9c, 0xc00193b000, 0x7, 0x0, 0xc00157bbd0, 0x0, 0x0)
        third_party/golang/gvisor/pkg/sentry/syscalls/linux/sys_file.go:100 +0x29c
google3/third_party/golang/gvisor/pkg/sentry/syscalls/linux/linux.Lstat(0xc000207500, 0x7fc1bc0daf30, 0x55b8260723d0, 0x55b8260723d0, 0x0, 0x0, 0x55b8260770d9, 0x0, 0x0, 0x0, ...)
        third_party/golang/gvisor/pkg/sentry/syscalls/linux/sys_stat.go:89 +0x103
google3/third_party/golang/gvisor/pkg/sentry/kernel/kernel.(*Task).executeSyscall(0xc000207500, 0x6, 0x7fc1bc0daf30, 0x55b8260723d0, 0x55b8260723d0, 0x0, 0x0, 0x55b8260770d9, 0x0, 0x0, ...)
        third_party/golang/gvisor/pkg/sentry/kernel/task_syscall.go:165 +0x2fc
google3/third_party/golang/gvisor/pkg/sentry/kernel/kernel.(*Task).doSyscallInvoke(0xc000207500, 0x6, 0x7fc1bc0daf30, 0x55b8260723d0, 0x55b8260723d0, 0x0, 0x0, 0x55b8260770d9, 0x0, 0x55b8260770d9)
        third_party/golang/gvisor/pkg/sentry/kernel/task_syscall.go:283 +0x69
google3/third_party/golang/gvisor/pkg/sentry/kernel/kernel.(*Task).doSyscallEnter(0xc000207500, 0x6, 0x7fc1bc0daf30, 0x55b8260723d0, 0x55b8260723d0, 0x0, 0x0, 0x55b8260770d9, 0x0, 0x0)
        third_party/golang/gvisor/pkg/sentry/kernel/task_syscall.go:244 +0x96
google3/third_party/golang/gvisor/pkg/sentry/kernel/kernel.(*Task).doSyscall(0xc000207500, 0x2, 0xc0009d8058)
        third_party/golang/gvisor/pkg/sentry/kernel/task_syscall.go:219 +0x13d
google3/third_party/golang/gvisor/pkg/sentry/kernel/kernel.(*runApp).execute(0x0, 0xc000207500, 0xee6100, 0x0)
        third_party/golang/gvisor/pkg/sentry/kernel/task_run.go:225 +0x1089
google3/third_party/golang/gvisor/pkg/sentry/kernel/kernel.(*Task).run(0xc000207500, 0x6f9)
        third_party/golang/gvisor/pkg/sentry/kernel/task_run.go:92 +0x194
created by google3/third_party/golang/gvisor/pkg/sentry/kernel/kernel.(*Task).Start
        third_party/golang/gvisor/pkg/sentry/kernel/task_start.go:286 +0xfe

goroutine 1 [semacquire, 16 minutes]:
sync.runtime_Semacquire(0xc0002a8674)
        third_party/go/gc/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc0002a8674)
        third_party/go/gc/src/sync/waitgroup.go:130 +0x65
google3/third_party/golang/gvisor/pkg/sentry/kernel/kernel.(*Kernel).WaitExited(...)
        third_party/golang/gvisor/pkg/sentry/kernel/kernel.go:861
google3/third_party/golang/gvisor/runsc/boot/boot.(*Loader).WaitExit(0xc00016d2c0, 0x0, 0x0)
        third_party/golang/gvisor/runsc/boot/loader.go:800 +0x3b
google3/third_party/golang/gvisor/runsc/cmd/cmd.(*Boot).Execute(0xc00015a6c0, 0xef7960, 0xc000138000, 0xc00015a870, 0xc000248e60, 0x2, 0x2, 0x0)
        third_party/golang/gvisor/runsc/cmd/boot.go:252 +0xc80
google3/third_party/golang/subcommands/subcommands.(*Commander).Execute(0xc0001998f0, 0xef7960, 0xc000138000, 0xc000248e60, 0x2, 0x2, 0xc000034000)
        third_party/golang/subcommands/subcommands.go:143 +0x303
google3/third_party/golang/subcommands/subcommands.Execute(...)
        third_party/golang/subcommands/subcommands.go:421
main.main()
        third_party/golang/gvisor/runsc/main.go:243 +0x13f3

goroutine 4 [syscall]:
os/signal.signal_recv(0xeeba60)
        third_party/go/gc/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()

[hbhasker]

Okay I think an easy repro is to edit a file inside the sandbox and save it. Then edit the same file from outside. This triggers the panic listed above.

[prattmic]

Then edit the same file from outside.

This violates the --file-access=exclusive (default) contract. Are you using --file-access=shared?

https://gvisor.dev/docs/user_guide/filesystem/

[hbhasker]

Ah that makes sense. I wasn't using --file-access=shared. I guess panic is appropriate in this case?

[nlacasse]

Yeah, in this case the caching in the FS layer is out-of-sync from the actual data, and is truly in an inconsistent state (through no fault of the sandbox). I'm not sure it's possible to recover from that inconsistency, so a panic seems appropriate.