Release VMAccess before preparing to checkpoint

The thread that hooks into the JIT to start preparing for
checkpoint has VMAccess. However, any compilation threads currently
compiling will request Exclusive VMAccess at some point during the
compilation to add the code cache to the artifact manager. Because the
hook thread waits for all compilations in flight to complete, a deadlock
condition arises as the compilation thread waits for Exclusive VMAccess
that it can't acquire because the hook thread has VMAccess, while the
hook thread waits for the compilation thread to notify it that it will
suspend itself.

This commit fixes this by releasing VMAccess before preparing to
checkpoint, and requires it before returning to the VM. This is ok
because all other java threads have been halted at this point.

Signed-off-by: Irwin D'Souza <[email protected]>

runtime/compiler/control/CompilationThread.cpp

@@ -2613,13 +2613,40 @@ void TR::CompilationInfo::resumeCompilationThread()
    }
 
 #if defined(J9VM_OPT_CRIU_SUPPORT)
+
+class ReleaseVMAccessAndAcquireMonitor
+   {
+   public:
+   ReleaseVMAccessAndAcquireMonitor(J9VMThread *vmThread, TR::Monitor *monitor)
+      : _hadVMAccess(vmThread->publicFlags & J9_PUBLIC_FLAGS_VM_ACCESS),
+        _vmThread(vmThread),
+        _monitor(monitor)
+      {
+      if (_hadVMAccess)
+         releaseVMAccessNoSuspend(_vmThread);
+      _monitor->enter();
+      }
+
+   ~ReleaseVMAccessAndAcquireMonitor()
+      {
+      _monitor->exit();
+      if (_hadVMAccess)
+         acquireVMAccessNoSuspend(_vmThread);
+      }
+
+   private:
+   bool         _hadVMAccess;
+   J9VMThread  *_vmThread;
+   TR::Monitor *_monitor;
+   };
+
 void TR::CompilationInfo::prepareForCheckpoint()
    {
    J9JavaVM   *vm       = _jitConfig->javaVM;
    J9VMThread *vmThread = vm->internalVMFunctions->currentVMThread(vm);
 
    {
-   OMR::CriticalSection suspendCompThreadsForCheckpoint(getCompilationMonitor());
+   ReleaseVMAccessAndAcquireMonitor suspendCompThreadsForCheckpoint(vmThread, getCompilationMonitor());
 
    if (shouldCheckpointBeInterrupted())
       return;