diff --git a/runtime/jilgen/jilconsts.c b/runtime/jilgen/jilconsts.c index 222963d01f0..ce832f13aed 100644 --- a/runtime/jilgen/jilconsts.c +++ b/runtime/jilgen/jilconsts.c @@ -762,6 +762,7 @@ writeConstants(OMRPortLibrary *OMRPORTLIB, IDATA fd) writeConstant(OMRPORTLIB, fd, "J9TR_bcloop_enter_method_monitor", J9_BCLOOP_ENTER_METHOD_MONITOR) | writeConstant(OMRPORTLIB, fd, "J9TR_bcloop_report_method_enter", J9_BCLOOP_REPORT_METHOD_ENTER) | writeConstant(OMRPORTLIB, fd, "J9TR_bcloop_exit_interpreter", J9_BCLOOP_EXIT_INTERPRETER) | + writeConstant(OMRPORTLIB, fd, "J9TR_bcloop_reenter_interpreter", J9_BCLOOP_REENTER_INTERPRETER) | writeConstant(OMRPORTLIB, fd, "J9TR_MethodNotCompiledBit", J9_STARTPC_NOT_TRANSLATED) | writeConstant(OMRPORTLIB, fd, "J9TR_InterpVTableOffset", J9JIT_INTERP_VTABLE_OFFSET) | writeConstant(OMRPORTLIB, fd, "J9TR_RequiredClassAlignment", J9_REQUIRED_CLASS_ALIGNMENT) | diff --git a/runtime/oti/VMHelpers.hpp b/runtime/oti/VMHelpers.hpp index 67267588c48..2d2e4f8bc32 100644 --- a/runtime/oti/VMHelpers.hpp +++ b/runtime/oti/VMHelpers.hpp @@ -803,7 +803,7 @@ class VM_VMHelpers static VMINLINE bool immediateAsyncPending(J9VMThread *currentThread) { - return (0 != (currentThread->publicFlags & J9_PUBLIC_FLAGS_POP_FRAMES_INTERRUPT)); + return J9_ARE_ANY_BITS_SET(currentThread->publicFlags, J9_PUBLIC_FLAGS_POP_FRAMES_INTERRUPT); } /** @@ -2170,6 +2170,34 @@ class VM_VMHelpers return result; } #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */ + + /** + * Query and reset the current thread's interpreter re-entry flag. + * + * @param[in] currentThread the current J9VMThread + * + * @return true if re-entry requested, false if not + */ + static VMINLINE bool + interpreterReentryRequested(J9VMThread *currentThread) + { + bool rc = J9_ARE_ANY_BITS_SET(currentThread->privateFlags2, J9_PRIVATE_FLAGS2_REENTER_INTERPRETER); + currentThread->privateFlags2 &= ~(UDATA)J9_PRIVATE_FLAGS2_REENTER_INTERPRETER; + return rc; + } + + /** + * Request interpreter re-entry on a J9VMThread. The target thread must either be + * the current thread or not have VM access (e.g. halted by exclusive VM access). + * + * @param[in] targetThread the target J9VMThread + */ + static VMINLINE void + requestInterpreterReentry(J9VMThread *targetThread) + { + targetThread->privateFlags2 |= J9_PRIVATE_FLAGS2_REENTER_INTERPRETER; + indicateAsyncMessagePending(targetThread); + } }; #endif /* VMHELPERS_HPP_ */ diff --git a/runtime/oti/j9consts.h b/runtime/oti/j9consts.h index cbb51608fe2..0a5a34c4d0a 100644 --- a/runtime/oti/j9consts.h +++ b/runtime/oti/j9consts.h @@ -181,6 +181,7 @@ extern "C" { #define J9_PRIVATE_FLAGS2_CHECK_PACKAGE_ACCESS 0x2 #define J9_PRIVATE_FLAGS2_UNSAFE_HANDLE_SIGBUS 0x4 #define J9_PRIVATE_FLAGS2_ASYNC_GET_CALL_TRACE 0x8 /* DO NOT set this anywhere but AsyncGetCallTrace */ +#define J9_PRIVATE_FLAGS2_REENTER_INTERPRETER 0x10 #define J9_PUBLIC_FLAGS_HALT_THREAD_EXCLUSIVE 0x1 #define J9_PUBLIC_FLAGS_DEBUG_VM_ACCESS 0x2 @@ -720,6 +721,7 @@ extern "C" { #if JAVA_SPEC_VERSION >= 16 #define J9_BCLOOP_N2I_TRANSITION 0x18 #endif /* JAVA_SPEC_VERSION >= 16 */ +#define J9_BCLOOP_REENTER_INTERPRETER 0x19 #define J9_RAS_METHOD_UNSEEN 0x0 #define J9_RAS_METHOD_SEEN 0x1 diff --git a/runtime/vm/BytecodeInterpreter.hpp b/runtime/vm/BytecodeInterpreter.hpp index 15f9c3ce165..67764011b2f 100644 --- a/runtime/vm/BytecodeInterpreter.hpp +++ b/runtime/vm/BytecodeInterpreter.hpp @@ -189,6 +189,25 @@ class INTERPRETER_CLASS * Function members */ private: + + VMINLINE bool + interpreterReentryRequested() + { +#if defined(J9VM_OPT_CRIU_SUPPORT) + return VM_VMHelpers::interpreterReentryRequested(_currentThread); +#else /* J9VM_OPT_CRIU_SUPPORT */ + return false; +#endif /* J9VM_OPT_CRIU_SUPPORT */ + } + + VMINLINE VM_BytecodeAction + reenterInterpreter(UDATA reentryAction) + { + _currentThread->returnValue = reentryAction; + _nextAction = J9_BCLOOP_REENTER_INTERPRETER; + return GOTO_DONE; + } + #if defined(J9VM_OPT_METHOD_HANDLE) /** * Run a methodHandle using the MethodHandle interpreter/ @@ -1447,66 +1466,74 @@ class INTERPRETER_CLASS VMINLINE VM_BytecodeAction checkAsync(REGISTER_ARGS_LIST) { - /* The current stack frame may be one of the following: - * - * 1) Special frame - * - * 1a) INL frame (for an INL which returns void) - * - * No frame build is required, and the restoreSpecialStackFrameAndDrop will remove the - * argument from the stack before resuming execution. - * - * 1b) An immediate async is pending - * - * The stack is already in a walkable state, and the restore and executeCurrentBytecode - * will not take place. - * - * 2) Bytecode or JNI call-in frame - * - * Build a generic special frame, tear it down when done. - */ VM_BytecodeAction rc = EXECUTE_BYTECODE; - bool inlFrame = false; - UDATA *bp = NULL; - if ((UDATA)_pc <= J9SF_MAX_SPECIAL_FRAME_TYPE) { - if (J9SF_FRAME_TYPE_NATIVE_METHOD == (UDATA)_pc) { - inlFrame = true; - bp = ((UDATA*)(((J9SFNativeMethodFrame*)((UDATA)_sp + (UDATA)_literals)) + 1)) - 1; - } + + /* Check for interpreter re-entry first because we will come back to checkAsync + * immediately upon re-entry to the interpreter. + */ + if (interpreterReentryRequested()) { + rc = reenterInterpreter(J9_BCLOOP_CHECK_ASYNC); } else { - buildGenericSpecialStackFrame(REGISTER_ARGS, 0); - bp = _arg0EA; - } - UDATA relativeBP = bp - _arg0EA; - updateVMStruct(REGISTER_ARGS); - UDATA action = javaCheckAsyncMessages(_currentThread, TRUE); - VMStructHasBeenUpdated(REGISTER_ARGS); - switch(action) { - case J9_CHECK_ASYNC_THROW_EXCEPTION: - rc = GOTO_THROW_CURRENT_EXCEPTION; - break; -#if defined(DEBUG_VERSION) - case J9_CHECK_ASYNC_POP_FRAMES: - rc = HANDLE_POP_FRAMES; - break; -#endif - default: - restoreSpecialStackFrameAndDrop(REGISTER_ARGS, _arg0EA + relativeBP); - if (inlFrame) { - _pc += 3; + /* The current stack frame may be one of the following: + * + * 1) Special frame + * + * 1a) INL frame (for an INL which returns void) + * + * No frame build is required, and the restoreSpecialStackFrameAndDrop will remove the + * argument from the stack before resuming execution. + * + * 1b) An immediate async is pending + * + * The stack is already in a walkable state, and the restore and executeCurrentBytecode + * will not take place. + * + * 2) Bytecode or JNI call-in frame + * + * Build a generic special frame, tear it down when done. + */ + bool inlFrame = false; + UDATA *bp = NULL; + if ((UDATA)_pc <= J9SF_MAX_SPECIAL_FRAME_TYPE) { + if (J9SF_FRAME_TYPE_NATIVE_METHOD == (UDATA)_pc) { + inlFrame = true; + bp = ((UDATA*)(((J9SFNativeMethodFrame*)((UDATA)_sp + (UDATA)_literals)) + 1)) - 1; + } + } else { + buildGenericSpecialStackFrame(REGISTER_ARGS, 0); + bp = _arg0EA; + } + UDATA relativeBP = bp - _arg0EA; + updateVMStruct(REGISTER_ARGS); + UDATA action = javaCheckAsyncMessages(_currentThread, TRUE); + VMStructHasBeenUpdated(REGISTER_ARGS); + switch(action) { + case J9_CHECK_ASYNC_THROW_EXCEPTION: + rc = GOTO_THROW_CURRENT_EXCEPTION; + break; + #if defined(DEBUG_VERSION) + case J9_CHECK_ASYNC_POP_FRAMES: + rc = HANDLE_POP_FRAMES; + break; + #endif + default: + restoreSpecialStackFrameAndDrop(REGISTER_ARGS, _arg0EA + relativeBP); + if (inlFrame) { + _pc += 3; + } + break; } - break; } return rc; } VMINLINE bool immediateAsyncPending() { -#if defined(DEBUG_VERSION) +#if defined(DEBUG_VERSION) || defined(J9VM_OPT_CRIU_SUPPORT) return VM_VMHelpers::immediateAsyncPending(_currentThread); -#else +#else /* defined(DEBUG_VERSION) || defined(J9VM_OPT_CRIU_SUPPORT) */ return false; -#endif +#endif /* defined(DEBUG_VERSION) || defined(J9VM_OPT_CRIU_SUPPORT) */ } VMINLINE VM_BytecodeAction @@ -1865,7 +1892,14 @@ class INTERPRETER_CLASS } #if defined(DO_HOOKS) rc = REPORT_METHOD_ENTER; -#endif + if (interpreterReentryRequested()) { + rc = reenterInterpreter(J9_BCLOOP_REPORT_METHOD_ENTER); + } +#else /* DO_HOOKS */ + if (interpreterReentryRequested()) { + rc = reenterInterpreter(J9_BCLOOP_EXECUTE_BYTECODE); + } +#endif /* DO_HOOKS */ } done: return rc; @@ -2341,6 +2375,9 @@ class INTERPRETER_CLASS *(U_32 *)_sp = (U_32)_currentThread->returnValue; } rc = EXECUTE_BYTECODE; + if (interpreterReentryRequested()) { + rc = reenterInterpreter(J9_BCLOOP_EXECUTE_BYTECODE); + } done: return rc; } diff --git a/runtime/vm/arm64cinterp.m4 b/runtime/vm/arm64cinterp.m4 index af2e1211715..b93d17f0ab8 100644 --- a/runtime/vm/arm64cinterp.m4 +++ b/runtime/vm/arm64cinterp.m4 @@ -49,6 +49,8 @@ FUNC_LABEL(cInterpreter): blr x28 cmp x0, {#}J9TR_bcloop_exit_interpreter beq .L_cInterpExit + cmp x0, {#}J9TR_bcloop_reenter_interpreter + beq FUNC_LABEL(cInterpreter RESTORE_PRESERVED_REGS RESTORE_FPLR SWITCH_TO_JAVA_STACK diff --git a/runtime/vm/armcinterp.m4 b/runtime/vm/armcinterp.m4 index 80b59ac992a..f6376eed706 100644 --- a/runtime/vm/armcinterp.m4 +++ b/runtime/vm/armcinterp.m4 @@ -42,6 +42,8 @@ cInterpreter: ldr r15,[r4,{#}J9TR_JavaVM_bytecodeLoop] cmp r0,{#}J9TR_bcloop_exit_interpreter beq .L_cInterpExit + cmp r0,{#}J9TR_bcloop_reenter_interpreter + beq cInterpreter RESTORE_PRESERVED_REGS RESTORE_LR SWITCH_TO_JAVA_STACK diff --git a/runtime/vm/pcinterp.m4 b/runtime/vm/pcinterp.m4 index 8146b8fac57..cfee7a14332 100644 --- a/runtime/vm/pcinterp.m4 +++ b/runtime/vm/pcinterp.m4 @@ -130,6 +130,8 @@ ifdef({ASM_J9VM_ENV_DATA64},{ CALL_INDIRECT cmpliaddr r3,J9TR_bcloop_exit_interpreter beq .L_cInterpExit + cmpliaddr r3,J9TR_bcloop_reenter_interpreter + beq .L_cInterpOnCStack RESTORE_PRESERVED_REGS RESTORE_LR SWITCH_TO_JAVA_STACK diff --git a/runtime/vm/riscvcinterp.m4 b/runtime/vm/riscvcinterp.m4 index c9a564fa3cc..fd566575642 100644 --- a/runtime/vm/riscvcinterp.m4 +++ b/runtime/vm/riscvcinterp.m4 @@ -70,6 +70,8 @@ cInterpreter: jalr ra, s8, 0 li s7, J9TR_bcloop_exit_interpreter beq a0, s7, .L_cInterpExit + li s7, J9TR_bcloop_reenter_interpreter + beq a0, s7, cInterpreter RESTORE_PRESERVED_REGS RESTORE_FPLR SWITCH_TO_JAVA_STACK diff --git a/runtime/vm/xcinterp.m4 b/runtime/vm/xcinterp.m4 index 60828d54c84..a45d7262b2b 100644 --- a/runtime/vm/xcinterp.m4 +++ b/runtime/vm/xcinterp.m4 @@ -168,6 +168,8 @@ C_FUNCTION_SYMBOL(cInterpreter): CALL_C_ADDR_WITH_VMTHREAD(uword ptr J9TR_JavaVM_bytecodeLoop[_rax],0) cmp _rax,J9TR_bcloop_exit_interpreter je SHORT_JMP cInterpExit + cmp _rax,J9TR_bcloop_reenter_interpreter + je SHORT_JMP C_FUNCTION_SYMBOL(cInterpreter) RESTORE_PRESERVED_REGS SWITCH_TO_JAVA_STACK jmp uword ptr J9TR_VMThread_tempSlot[_rbp] diff --git a/runtime/vm/zcinterp.m4 b/runtime/vm/zcinterp.m4 index eeb2aa7bc02..e548b516cd1 100644 --- a/runtime/vm/zcinterp.m4 +++ b/runtime/vm/zcinterp.m4 @@ -78,6 +78,9 @@ PLACE_LABEL(L_CINTERP) LHI_GPR r0,J9TR_bcloop_exit_interpreter CLR_GPR CRINT,r0 je LABEL_NAME(L_EXIT) + LHI_GPR r0,J9TR_bcloop_reenter_interpreter + CLR_GPR CRINT,r0 + je LABEL_NAME(L_CINTERP) RESTORE_LR RESTORE_PRESERVED_REGS_AND_SWITCH_TO_JAVA_STACK BRANCH_VIA_VMTHREAD(J9TR_VMThread_tempSlot)