From fea15f4e9a182215b1e098f003a5304f41d0fdf7 Mon Sep 17 00:00:00 2001 From: lateralusX Date: Sat, 11 Sep 2021 12:45:52 +0200 Subject: [PATCH] Deadlock during rundown using interpreter. Identified deadlock between finalizer thread and rundown enumerating all interpreter method. Since rundown will query for method name in callback when iterating interpreter methods, interp_jit_info_foreach, that might lead to additional loader activity. The hash map in interpreter keeping the methods is locked with default JIT memory manager, but since the callback might end up in mono_class_create_from_typedef that will take loader lock, we get the following lock order on that code path, memory manager->loader lock. Finalizer thread invokes OnThreadExiting using interpreter and that might end up with a reverse lock order, loader lock->memory manager on that code path, so these two have a potential to deadlock. This is not a problem under JIT or AOT since the JIT hash table is lock free therefore not causing any deadlocks due to lock order between memory manager and loader lock. Could be fixed by changing into a lock free hash table in interpreters for interp_code_hash might be to risky at this point. A more safe fix is to take a copy of the pointers while holding lock and then iterate using local copy (simple array of pointers). Since this method is only called during rundown, only when using interpreter, and only include the pointers (InterpMethod *) we use with the callback, it will have some temporary memory impact (allocating an array of pointers), but will mitigate the deadlock since we can safely call iterator callback without holding the lock. It will also improve interpreter performance in situations where we run session rundown, since lock will be held a much shorter amount of time. --- src/mono/mono/mini/interp/interp.c | 36 +++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/mono/mono/mini/interp/interp.c b/src/mono/mono/mini/interp/interp.c index a79411b5ee932..7734c0631cb73 100644 --- a/src/mono/mono/mini/interp/interp.c +++ b/src/mono/mono/mini/interp/interp.c @@ -7435,28 +7435,44 @@ interp_invalidate_transformed (void) } typedef struct { - InterpJitInfoFunc func; - gpointer user_data; -} InterpJitInfoFuncUserData; + MonoJitInfo **jit_info_array; + gint size; + gint next; +} InterpCopyJitInfoFuncUserData; static void -interp_call_jit_info_func (gpointer imethod, gpointer user_data) +interp_copy_jit_info_func (gpointer imethod, gpointer user_data) { - InterpJitInfoFuncUserData *data = (InterpJitInfoFuncUserData *)user_data; - data->func (((InterpMethod *)imethod)->jinfo, data->user_data); + InterpCopyJitInfoFuncUserData *data = (InterpCopyJitInfoFuncUserData*)user_data; + if (data->next < data->size) + data->jit_info_array [data->next++] = ((InterpMethod *)imethod)->jinfo; } static void interp_jit_info_foreach (InterpJitInfoFunc func, gpointer user_data) { - InterpJitInfoFuncUserData data = {func, user_data}; + InterpCopyJitInfoFuncUserData copy_jit_info_data; // FIXME: Enumerate all memory managers MonoJitMemoryManager *jit_mm = get_default_jit_mm (); - jit_mm_lock (jit_mm); - mono_internal_hash_table_apply (&jit_mm->interp_code_hash, interp_call_jit_info_func, &data); - jit_mm_unlock (jit_mm); + // Can't keep memory manager lock while iterating and calling callback since it might take other locks + // causing poential deadlock situations. Instead, create copy of interpreter imethod jinfo pointers into + // plain array and use pointers from array when when running callbacks. + copy_jit_info_data.size = mono_atomic_load_i32 (&(jit_mm->interp_code_hash.num_entries)); + copy_jit_info_data.next = 0; + copy_jit_info_data.jit_info_array = (MonoJitInfo**) g_new (MonoJitInfo*, copy_jit_info_data.size); + if (copy_jit_info_data.jit_info_array) { + jit_mm_lock (jit_mm); + mono_internal_hash_table_apply (&jit_mm->interp_code_hash, interp_copy_jit_info_func, ©_jit_info_data); + jit_mm_unlock (jit_mm); + } + + if (copy_jit_info_data.jit_info_array) { + for (size_t i = 0; i < copy_jit_info_data.next; ++i) + func (copy_jit_info_data.jit_info_array [i], user_data); + g_free (copy_jit_info_data.jit_info_array); + } } static void