From ad11d6de4fee0ba763f4d303abb6782ca140a69d Mon Sep 17 00:00:00 2001 From: Jean-Claude Grenier Date: Fri, 21 Jul 2023 15:57:17 -0400 Subject: [PATCH 1/8] Implemented C# GC get heap size/ get used size --- src/coreclr/vm/mono/mono_coreclr.cpp | 11 +++++---- .../BaseEmbeddingApiTests.cs | 24 +++++++++++++++++++ unity/unity-embed-host/CoreCLRHost.cs | 13 ++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/coreclr/vm/mono/mono_coreclr.cpp b/src/coreclr/vm/mono/mono_coreclr.cpp index bab6991fa0df3..de3c7b1ef27cb 100644 --- a/src/coreclr/vm/mono/mono_coreclr.cpp +++ b/src/coreclr/vm/mono/mono_coreclr.cpp @@ -86,6 +86,8 @@ struct HostStruct gboolean (*class_is_valuetype)(MonoClass* klass); MonoException* (*exception_from_name_msg)(MonoImage* image, const char* name_space, const char* name, const char* msg); MonoReflectionField* (*field_get_object)(MonoDomain* domain, MonoClass* klass, MonoClassField* field); + gint64 (*gc_get_heap_size)(); + gint64 (*gc_get_used_size)(); intptr_t (*gchandle_get_target_v2)(intptr_t handleIn); uintptr_t (*gchandle_new_v2)(MonoObject* obj, gboolean pinned); uintptr_t (*gchandle_new_weakref_v2)(MonoObject* obj, gboolean track_resurrection); @@ -1292,11 +1294,10 @@ extern "C" EXPORT_API int EXPORT_CC mono_gc_collect_a_little () return 0; } +// Generated by UnityEmbedHost.Generator - Commit these changes extern "C" EXPORT_API gint64 EXPORT_CC mono_gc_get_heap_size() { - FCALL_CONTRACT; - // NOT CORRECT - return GCHeapUtilities::GetGCHeap()->GetTotalBytesInUse(); + return g_HostStruct->gc_get_heap_size(); } extern "C" EXPORT_API gint64 EXPORT_CC mono_gc_get_max_time_slice_ns () @@ -1305,10 +1306,10 @@ extern "C" EXPORT_API gint64 EXPORT_CC mono_gc_get_max_time_slice_ns () return 0; } +// Generated by UnityEmbedHost.Generator - Commit these changes extern "C" EXPORT_API gint64 EXPORT_CC mono_gc_get_used_size() { - FCALL_CONTRACT; - return GCHeapUtilities::GetGCHeap()->GetTotalBytesInUse(); + return g_HostStruct->gc_get_used_size(); } extern "C" EXPORT_API gboolean EXPORT_CC mono_gc_is_incremental () diff --git a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs index 9b5ce73f6b877..46cc26b8b40e0 100644 --- a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs +++ b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs @@ -783,6 +783,30 @@ public void UnityMethodIsGenericReturnsProperValue(Type objType, string methodNa Assert.That(isGeneric, Is.EqualTo(expectedResult)); } + [Test] + public void GcGetHeapSizeReturnsProperValue() + { + GC.Collect(); + long heapSize = ClrHost.gc_get_heap_size(); + Assert.NotZero(heapSize); + int[] data = new int[1024 * 1024 * 100]; + GC.Collect(); + long heapSizeAfterBigAlloc = ClrHost.gc_get_heap_size(); + Assert.Greater(heapSizeAfterBigAlloc, heapSize); + } + + [Test] + public void GcGetUsedSizeReturnsProperValue() + { + GC.Collect(); + long usedSize = ClrHost.gc_get_used_size(); + Assert.NotZero(usedSize); + int[] data = new int[1024 * 1024 * 100]; + GC.Collect(); + long usedSizeAfterBigAlloc = ClrHost.gc_get_used_size(); + Assert.Greater(usedSizeAfterBigAlloc, usedSize); + } + static List FlattenedArray(Array arr) { var result = new List(); diff --git a/unity/unity-embed-host/CoreCLRHost.cs b/unity/unity-embed-host/CoreCLRHost.cs index d5b0e470f80dc..e36158e384f1e 100644 --- a/unity/unity-embed-host/CoreCLRHost.cs +++ b/unity/unity-embed-host/CoreCLRHost.cs @@ -671,6 +671,19 @@ public static bool unity_mono_method_is_generic( return metBase.IsGenericMethodDefinition; } + [return: NativeCallbackType("gint64")] + public static long gc_get_heap_size() + { + var info = GC.GetGCMemoryInfo(); + return info.HeapSizeBytes; + } + + [return: NativeCallbackType("gint64")] + public static long gc_get_used_size() + { + return GC.GetTotalMemory(false); + } + static void Log(string message) { var bytes = System.Text.Encoding.UTF8.GetBytes(message); From b0452f4154f204983187ff0bbdcbc19eee7c2e91 Mon Sep 17 00:00:00 2001 From: Jean-Claude Grenier Date: Mon, 24 Jul 2023 11:50:47 -0400 Subject: [PATCH 2/8] Added a GC.KeepAlive as suggested in PR to stabilize the test. --- unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs index 46cc26b8b40e0..42d3a5f0c868a 100644 --- a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs +++ b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs @@ -793,6 +793,7 @@ public void GcGetHeapSizeReturnsProperValue() GC.Collect(); long heapSizeAfterBigAlloc = ClrHost.gc_get_heap_size(); Assert.Greater(heapSizeAfterBigAlloc, heapSize); + GC.KeepAlive(data); } [Test] @@ -805,6 +806,7 @@ public void GcGetUsedSizeReturnsProperValue() GC.Collect(); long usedSizeAfterBigAlloc = ClrHost.gc_get_used_size(); Assert.Greater(usedSizeAfterBigAlloc, usedSize); + GC.KeepAlive(data); } static List FlattenedArray(Array arr) From e17696c81ed1af8f73a335c0837ff20230788097 Mon Sep 17 00:00:00 2001 From: Jean-Claude Grenier Date: Mon, 24 Jul 2023 13:16:03 -0400 Subject: [PATCH 3/8] Forcing a synchronous aggressive GC first, as we are currently asserting that GC was not ready --- unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs index 42d3a5f0c868a..eaa02b587cc79 100644 --- a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs +++ b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs @@ -786,7 +786,7 @@ public void UnityMethodIsGenericReturnsProperValue(Type objType, string methodNa [Test] public void GcGetHeapSizeReturnsProperValue() { - GC.Collect(); + GC.Collect(0,GCCollectionMode.Forced, true); long heapSize = ClrHost.gc_get_heap_size(); Assert.NotZero(heapSize); int[] data = new int[1024 * 1024 * 100]; @@ -799,7 +799,7 @@ public void GcGetHeapSizeReturnsProperValue() [Test] public void GcGetUsedSizeReturnsProperValue() { - GC.Collect(); + GC.Collect(0,GCCollectionMode.Forced, true); long usedSize = ClrHost.gc_get_used_size(); Assert.NotZero(usedSize); int[] data = new int[1024 * 1024 * 100]; From 087a6010cf540f1d39a84a45bb897e1cb81815f8 Mon Sep 17 00:00:00 2001 From: Jean-Claude Grenier Date: Mon, 24 Jul 2023 15:50:14 -0400 Subject: [PATCH 4/8] Modified test to validate that the heap/used size is more than the dummy data allocated, and that used size is smaller than the total heap --- .../BaseEmbeddingApiTests.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs index eaa02b587cc79..fb057c1cd127f 100644 --- a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs +++ b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs @@ -789,10 +789,11 @@ public void GcGetHeapSizeReturnsProperValue() GC.Collect(0,GCCollectionMode.Forced, true); long heapSize = ClrHost.gc_get_heap_size(); Assert.NotZero(heapSize); - int[] data = new int[1024 * 1024 * 100]; + int dataSize = 1024 * 1024 * 100; + int[] data = new int[dataSize]; GC.Collect(); - long heapSizeAfterBigAlloc = ClrHost.gc_get_heap_size(); - Assert.Greater(heapSizeAfterBigAlloc, heapSize); + heapSize = ClrHost.gc_get_heap_size(); + Assert.Greater(heapSize, dataSize * sizeof(int)); GC.KeepAlive(data); } @@ -802,10 +803,13 @@ public void GcGetUsedSizeReturnsProperValue() GC.Collect(0,GCCollectionMode.Forced, true); long usedSize = ClrHost.gc_get_used_size(); Assert.NotZero(usedSize); - int[] data = new int[1024 * 1024 * 100]; + int dataSize = 1024 * 1024 * 100; + int[] data = new int[dataSize]; GC.Collect(); - long usedSizeAfterBigAlloc = ClrHost.gc_get_used_size(); - Assert.Greater(usedSizeAfterBigAlloc, usedSize); + usedSize = ClrHost.gc_get_used_size(); + long heapSize = ClrHost.gc_get_heap_size(); + Assert.Greater(usedSize, dataSize * sizeof(int)); + Assert.GreaterOrEqual(heapSize, usedSize); GC.KeepAlive(data); } From 2be1e9f7175d93a9f28629455c78f3ac08c09944 Mon Sep 17 00:00:00 2001 From: Jean-Claude Grenier Date: Tue, 25 Jul 2023 13:55:31 -0400 Subject: [PATCH 5/8] Added a timeout and a sleep to the gc test, to wait for the first gc to finish. --- .../UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs index fb057c1cd127f..442434b4a0e12 100644 --- a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs +++ b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs @@ -784,10 +784,16 @@ public void UnityMethodIsGenericReturnsProperValue(Type objType, string methodNa } [Test] + [Timeout(5000)] public void GcGetHeapSizeReturnsProperValue() { GC.Collect(0,GCCollectionMode.Forced, true); long heapSize = ClrHost.gc_get_heap_size(); + while (heapSize == 0) + { + Thread.Sleep(0); + heapSize = ClrHost.gc_get_used_size(); + } Assert.NotZero(heapSize); int dataSize = 1024 * 1024 * 100; int[] data = new int[dataSize]; @@ -798,10 +804,18 @@ public void GcGetHeapSizeReturnsProperValue() } [Test] + [Timeout(5000)] public void GcGetUsedSizeReturnsProperValue() { GC.Collect(0,GCCollectionMode.Forced, true); + long usedSize = ClrHost.gc_get_used_size(); + while (usedSize == 0) + { + Thread.Sleep(0); + usedSize = ClrHost.gc_get_used_size(); + } + Assert.NotZero(usedSize); int dataSize = 1024 * 1024 * 100; int[] data = new int[dataSize]; From 16514fa1d114fc3a41d31c470e669276f74fe545 Mon Sep 17 00:00:00 2001 From: Jean-Claude Grenier Date: Tue, 25 Jul 2023 16:06:22 -0400 Subject: [PATCH 6/8] Trying to add a GC.WaitForPendingFinalizers call, and a try catch to figure out what exception occured --- .../BaseEmbeddingApiTests.cs | 74 ++++++++++++------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs index 442434b4a0e12..075ec1ba29e67 100644 --- a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs +++ b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs @@ -787,44 +787,64 @@ public void UnityMethodIsGenericReturnsProperValue(Type objType, string methodNa [Timeout(5000)] public void GcGetHeapSizeReturnsProperValue() { - GC.Collect(0,GCCollectionMode.Forced, true); - long heapSize = ClrHost.gc_get_heap_size(); - while (heapSize == 0) + try { - Thread.Sleep(0); - heapSize = ClrHost.gc_get_used_size(); + GC.Collect(0,GCCollectionMode.Forced, true); + GC.WaitForPendingFinalizers(); + + long heapSize = ClrHost.gc_get_heap_size(); + while (heapSize == 0) + { + Thread.Sleep(0); + heapSize = ClrHost.gc_get_used_size(); + } + Assert.NotZero(heapSize); + int dataSize = 1024 * 1024 * 100; + int[] data = new int[dataSize]; + GC.Collect(); + heapSize = ClrHost.gc_get_heap_size(); + Assert.Greater(heapSize, dataSize * sizeof(int)); + GC.KeepAlive(data); } - Assert.NotZero(heapSize); - int dataSize = 1024 * 1024 * 100; - int[] data = new int[dataSize]; - GC.Collect(); - heapSize = ClrHost.gc_get_heap_size(); - Assert.Greater(heapSize, dataSize * sizeof(int)); - GC.KeepAlive(data); + catch (Exception e) + { + Console.WriteLine(e); + throw; + } + } [Test] [Timeout(5000)] public void GcGetUsedSizeReturnsProperValue() { - GC.Collect(0,GCCollectionMode.Forced, true); - - long usedSize = ClrHost.gc_get_used_size(); - while (usedSize == 0) + try { - Thread.Sleep(0); + GC.Collect(0,GCCollectionMode.Forced, true); + GC.WaitForPendingFinalizers(); + + long usedSize = ClrHost.gc_get_used_size(); + while (usedSize == 0) + { + Thread.Sleep(0); + usedSize = ClrHost.gc_get_used_size(); + } + + Assert.NotZero(usedSize); + int dataSize = 1024 * 1024 * 100; + int[] data = new int[dataSize]; + GC.Collect(); usedSize = ClrHost.gc_get_used_size(); + long heapSize = ClrHost.gc_get_heap_size(); + Assert.Greater(usedSize, dataSize * sizeof(int)); + Assert.GreaterOrEqual(heapSize, usedSize); + GC.KeepAlive(data); + } + catch (Exception e) + { + Console.WriteLine(e); + throw; } - - Assert.NotZero(usedSize); - int dataSize = 1024 * 1024 * 100; - int[] data = new int[dataSize]; - GC.Collect(); - usedSize = ClrHost.gc_get_used_size(); - long heapSize = ClrHost.gc_get_heap_size(); - Assert.Greater(usedSize, dataSize * sizeof(int)); - Assert.GreaterOrEqual(heapSize, usedSize); - GC.KeepAlive(data); } static List FlattenedArray(Array arr) From 96b644dd29dfdfcd2f083e1ebcb869210591b792 Mon Sep 17 00:00:00 2001 From: Jean-Claude Grenier Date: Wed, 26 Jul 2023 00:45:14 -0400 Subject: [PATCH 7/8] boosted timeout to 50 seconds as 5 seems too short in some cases. modified gc_get_used_size to use GC.GetGCMemoryInfo as there seems to be random discrepencies between GC.GetGCMemoryInfo values and GC.GetTotalMemory data (it may also be a race condition issue) --- unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs | 4 ++-- unity/unity-embed-host/CoreCLRHost.cs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs index 075ec1ba29e67..64baaab5724da 100644 --- a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs +++ b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs @@ -784,7 +784,7 @@ public void UnityMethodIsGenericReturnsProperValue(Type objType, string methodNa } [Test] - [Timeout(5000)] + [Timeout(50000)] public void GcGetHeapSizeReturnsProperValue() { try @@ -815,7 +815,7 @@ public void GcGetHeapSizeReturnsProperValue() } [Test] - [Timeout(5000)] + [Timeout(50000)] public void GcGetUsedSizeReturnsProperValue() { try diff --git a/unity/unity-embed-host/CoreCLRHost.cs b/unity/unity-embed-host/CoreCLRHost.cs index e36158e384f1e..c1075f262de64 100644 --- a/unity/unity-embed-host/CoreCLRHost.cs +++ b/unity/unity-embed-host/CoreCLRHost.cs @@ -616,7 +616,7 @@ public static bool unity_class_is_abstract( Type t = klass.TypeFromHandleIntPtr(); return t.IsAbstract; } - + private static ConcurrentDictionary s_isBlittableCache = new ConcurrentDictionary(); [return: NativeCallbackType("gboolean")] public static bool class_is_blittable( @@ -681,7 +681,9 @@ public static long gc_get_heap_size() [return: NativeCallbackType("gint64")] public static long gc_get_used_size() { - return GC.GetTotalMemory(false); + var info = GC.GetGCMemoryInfo(); + var heapSz = info.HeapSizeBytes; + return heapSz - info.FragmentedBytes; } static void Log(string message) From 3eb130eb0de99ca382eff9e2e72446fce66201c0 Mon Sep 17 00:00:00 2001 From: Jean-Claude Grenier Date: Wed, 26 Jul 2023 01:53:41 -0400 Subject: [PATCH 8/8] Removed the test's try/catch (the exception was just the timeout being triggered) Removed test to check that used memory is less or equal to total memory, since this is likely to cause instabilities if the GC kicks in between the calls, as it seems like it happened in prior tests. --- .../BaseEmbeddingApiTests.cs | 73 +++++++------------ 1 file changed, 27 insertions(+), 46 deletions(-) diff --git a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs index 64baaab5724da..c1146359856f3 100644 --- a/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs +++ b/unity/UnityEmbedHost.Tests/BaseEmbeddingApiTests.cs @@ -787,64 +787,45 @@ public void UnityMethodIsGenericReturnsProperValue(Type objType, string methodNa [Timeout(50000)] public void GcGetHeapSizeReturnsProperValue() { - try - { - GC.Collect(0,GCCollectionMode.Forced, true); - GC.WaitForPendingFinalizers(); + GC.Collect(0,GCCollectionMode.Forced, true); + GC.WaitForPendingFinalizers(); - long heapSize = ClrHost.gc_get_heap_size(); - while (heapSize == 0) - { - Thread.Sleep(0); - heapSize = ClrHost.gc_get_used_size(); - } - Assert.NotZero(heapSize); - int dataSize = 1024 * 1024 * 100; - int[] data = new int[dataSize]; - GC.Collect(); - heapSize = ClrHost.gc_get_heap_size(); - Assert.Greater(heapSize, dataSize * sizeof(int)); - GC.KeepAlive(data); - } - catch (Exception e) + long heapSize = ClrHost.gc_get_heap_size(); + while (heapSize == 0) { - Console.WriteLine(e); - throw; + Thread.Sleep(0); + heapSize = ClrHost.gc_get_used_size(); } - + Assert.NotZero(heapSize); + int dataSize = 1024 * 1024 * 100; + int[] data = new int[dataSize]; + GC.Collect(); + heapSize = ClrHost.gc_get_heap_size(); + Assert.Greater(heapSize, dataSize * sizeof(int)); + GC.KeepAlive(data); } [Test] [Timeout(50000)] public void GcGetUsedSizeReturnsProperValue() { - try - { - GC.Collect(0,GCCollectionMode.Forced, true); - GC.WaitForPendingFinalizers(); + GC.Collect(0,GCCollectionMode.Forced, true); + GC.WaitForPendingFinalizers(); - long usedSize = ClrHost.gc_get_used_size(); - while (usedSize == 0) - { - Thread.Sleep(0); - usedSize = ClrHost.gc_get_used_size(); - } - - Assert.NotZero(usedSize); - int dataSize = 1024 * 1024 * 100; - int[] data = new int[dataSize]; - GC.Collect(); - usedSize = ClrHost.gc_get_used_size(); - long heapSize = ClrHost.gc_get_heap_size(); - Assert.Greater(usedSize, dataSize * sizeof(int)); - Assert.GreaterOrEqual(heapSize, usedSize); - GC.KeepAlive(data); - } - catch (Exception e) + long usedSize = ClrHost.gc_get_used_size(); + while (usedSize == 0) { - Console.WriteLine(e); - throw; + Thread.Sleep(0); + usedSize = ClrHost.gc_get_used_size(); } + + Assert.NotZero(usedSize); + int dataSize = 1024 * 1024 * 100; + int[] data = new int[dataSize]; + GC.Collect(); + usedSize = ClrHost.gc_get_used_size(); + Assert.Greater(usedSize, dataSize * sizeof(int)); + GC.KeepAlive(data); } static List FlattenedArray(Array arr)