[ VM ] Add support for heap sampling profiler

This CL introduces new embedding APIs for supporting heap sample
profiling. A registered sampling callback is invoked approximately every
N bytes based on an exponential distribution, providing information
about the isolate group the allocation occurred in, the user visible
name of the allocated object type, a weak persistent handle to the
allocated object, and the size of the allocation.

Sampling is triggered using artificial TLAB boundaries to cause
allocations to be sampled to take the allocation slow path where the
registered callback can be invoked with the allocation information.

Only new space allocations are currently traced, with old space
allocation support to be added in a future CL.

TEST=Dart_HeapSampling

Change-Id: I22bcdeec6e823bc1ab44898d4c596fbed7169fa1
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/264520
Commit-Queue: Ben Konyi <[email protected]>
Reviewed-by: Siva Annamalai <[email protected]>
Reviewed-by: Ryan Macnak <[email protected]>

runtime/include/dart_api.h

@@ -1270,6 +1270,52 @@ DART_EXPORT void Dart_KillIsolate(Dart_Isolate isolate);
  */
 DART_EXPORT void Dart_NotifyIdle(int64_t deadline);
 
+typedef void (*Dart_HeapSamplingCallback)(void* isolate_group_data,
+                                          Dart_Handle cls_name,
+                                          Dart_WeakPersistentHandle obj,
+                                          uintptr_t size);
+
+/**
+ * Starts the heap sampling profiler for each thread in the VM.
+ */
+DART_EXPORT void Dart_EnableHeapSampling();
+
+/*
+ * Stops the heap sampling profiler for each thread in the VM.
+ */
+DART_EXPORT void Dart_DisableHeapSampling();
+
+/*
+ * Registers a callback that is invoked once per sampled allocation.
+ *
+ * Important notes:
+ *
+ * - When invoked, |cls_name| will be a handle to a Dart String representing
+ *   the class name of the allocated object. This handle is stable and can be
+ *   used as an identifier as it has the lifetime of its isolate group.
+ *
+ * - |obj| is a weak persistent handle to the object which caused the
+ *   allocation. The value of this handle will be set to null when the object is
+ *   garbage collected. |obj| should only be used to determine whether the
+ *   object has been collected as there is no guarantee that it has been fully
+ *   initialized. This handle should eventually be freed with
+ *   Dart_DeleteWeakPersistentHandle once the embedder no longer needs it.
+ *
+ * - The provided callback must not call into the VM and should do as little
+ *   work as possible to avoid performance penalities.
+ */
+DART_EXPORT void Dart_RegisterHeapSamplingCallback(
+    Dart_HeapSamplingCallback callback);
+
+/*
+ * Sets the average heap sampling rate based on a number of |bytes| for each
+ * thread.
+ *
+ * In other words, approximately every |bytes| allocated will create a sample.
+ * Defaults to 512 KiB.
+ */
+DART_EXPORT void Dart_SetHeapSamplingPeriod(intptr_t bytes);
+
 /**
  * Notifies the VM that the embedder expects the application's working set has
  * recently shrunk significantly and is not expected to rise in the near future.

runtime/vm/class_table.h

@@ -32,6 +32,7 @@ class JSONStream;
 template <typename T>
 class MallocGrowableArray;
 class ObjectPointerVisitor;
+class PersistentHandle;
 
 // A 64-bit bitmap describing unboxed fields in a class.
 //
@@ -424,6 +425,13 @@ class ClassTable : public MallocAllocated {
         classes_.GetColumn<kAllocationTracingStateIndex>());
   }
 
+  PersistentHandle* UserVisibleNameFor(intptr_t cid) {
+    return classes_.At<kClassNameIndex>(cid);
+  }
+
+  void SetUserVisibleNameFor(intptr_t cid, PersistentHandle* name) {
+    classes_.At<kClassNameIndex>(cid) = name;
+  }
 #else
   void UpdateCachedAllocationTracingStateTablePointer() {}
 #endif  // !defined(PRODUCT)
@@ -542,7 +550,8 @@ class ClassTable : public MallocAllocated {
     kSizeIndex,
     kUnboxedFieldBitmapIndex,
 #if !defined(PRODUCT)
-    kAllocationTracingStateIndex
+    kAllocationTracingStateIndex,
+    kClassNameIndex,
 #endif
   };
 
@@ -551,7 +560,8 @@ class ClassTable : public MallocAllocated {
                   ClassPtr,
                   uint32_t,
                   UnboxedFieldBitmap,
-                  uint8_t>
+                  uint8_t,
+                  PersistentHandle*>
       classes_;
 #else
   CidIndexedTable<ClassIdTagType, ClassPtr, uint32_t, UnboxedFieldBitmap>

runtime/vm/dart_api_impl.cc

@@ -1826,6 +1826,45 @@ DART_EXPORT void Dart_NotifyDestroyed() {
   T->heap()->NotifyDestroyed();
 }
 
+DART_EXPORT void Dart_EnableHeapSampling() {
+#if !defined(PRODUCT)
+  IsolateGroup::ForEach([&](IsolateGroup* group) {
+    group->thread_registry()->ForEachThread(
+        [&](Thread* thread) { thread->heap_sampler().Enable(true); });
+  });
+#endif
+}
+
+DART_EXPORT void Dart_DisableHeapSampling() {
+#if !defined(PRODUCT)
+  IsolateGroup::ForEach([&](IsolateGroup* group) {
+    group->thread_registry()->ForEachThread(
+        [&](Thread* thread) { thread->heap_sampler().Enable(false); });
+  });
+#endif
+}
+
+DART_EXPORT void Dart_RegisterHeapSamplingCallback(
+    Dart_HeapSamplingCallback callback) {
+#if !defined(PRODUCT)
+  IsolateGroup::ForEach([&](IsolateGroup* group) {
+    group->thread_registry()->ForEachThread([&](Thread* thread) {
+      thread->heap_sampler().SetSamplingCallback(callback);
+    });
+  });
+#endif
+}
+
+DART_EXPORT void Dart_SetHeapSamplingPeriod(intptr_t bytes) {
+#if !defined(PRODUCT)
+  IsolateGroup::ForEach([&](IsolateGroup* group) {
+    group->thread_registry()->ForEachThread([&](Thread* thread) {
+      thread->heap_sampler().SetSamplingInterval(bytes);
+    });
+  });
+#endif
+}
+
 DART_EXPORT void Dart_NotifyLowMemory() {
   API_TIMELINE_BEGIN_END(Thread::Current());
   Page::ClearCache();

runtime/vm/dart_api_impl_test.cc

@@ -10494,6 +10494,96 @@ TEST_CASE(DartAPI_UserTags) {
       "Dart_SetCurrentUserTag expects argument 'user_tag' to be non-null");
 }
 
+void* last_isolate_group_data = nullptr;
+Dart_PersistentHandle last_allocation_cls = nullptr;
+intptr_t heap_samples = 0;
+
+void HeapSamplingCallback(void* isolate_group_data,
+                          Dart_PersistentHandle cls_type,
+                          Dart_WeakPersistentHandle obj,
+                          uintptr_t size) {
+  last_isolate_group_data = isolate_group_data;
+  last_allocation_cls = cls_type;
+  heap_samples++;
+}
+
+TEST_CASE(DartAPI_HeapSampling) {
+  Dart_RegisterHeapSamplingCallback(HeapSamplingCallback);
+
+  Dart_EnableHeapSampling();
+  // Start with sampling on every byte allocated.
+  Dart_SetHeapSamplingPeriod(1);
+
+  auto isolate_group_data = Dart_CurrentIsolateGroupData();
+  // Some simple allocations
+  USE(Dart_NewList(100));
+
+  const char* name = nullptr;
+  Dart_Handle result = Dart_StringToCString(last_allocation_cls, &name);
+  EXPECT_VALID(result);
+
+  EXPECT(heap_samples > 0);
+  EXPECT_STREQ("List", name);
+  EXPECT_EQ(last_isolate_group_data, isolate_group_data);
+
+  heap_samples = 0;
+  USE(Dart_NewStringFromCString("Foo"));
+  result = Dart_StringToCString(last_allocation_cls, &name);
+  EXPECT_VALID(result);
+  EXPECT(heap_samples > 0);
+  EXPECT_STREQ("String", name);
+  EXPECT_EQ(last_isolate_group_data, isolate_group_data);
+
+  // Increase the sampling period and check that we don't sample each
+  // allocation. This should cause samples to be collected for approximately
+  // every 1KiB allocated.
+  Dart_SetHeapSamplingPeriod(1 << 10);
+  heap_samples = 0;
+
+  const intptr_t kNumAllocations = 1000;
+  for (intptr_t i = 0; i < kNumAllocations; ++i) {
+    USE(Dart_NewList(10));
+  }
+  EXPECT(heap_samples > 0);
+  EXPECT(heap_samples < kNumAllocations);
+
+  heap_samples = 0;
+  last_allocation_cls = nullptr;
+  const char* kScriptChars = R"(
+    foo() {
+      final list = [];
+      for (int i = 0; i < 1000; ++i) {
+        list.add(List.filled(100, 0));
+      }
+    }
+    )";
+  Dart_DisableHeapSampling();
+  Dart_Handle lib = TestCase::LoadTestScript(kScriptChars, nullptr);
+  EXPECT_VALID(lib);
+  Dart_EnableHeapSampling();
+  result = Dart_Invoke(lib, NewString("foo"), 0, nullptr);
+  EXPECT_VALID(result);
+  EXPECT(heap_samples > 0);
+  EXPECT(heap_samples < kNumAllocations);
+
+  Dart_DisableHeapSampling();
+
+  // Sampling on every byte allocated.
+  Dart_SetHeapSamplingPeriod(1);
+
+  // Ensure no more samples are collected.
+  heap_samples = 0;
+  last_allocation_cls = nullptr;
+  last_isolate_group_data = nullptr;
+  USE(Dart_NewList(10));
+  EXPECT_EQ(heap_samples, 0);
+  EXPECT_EQ(last_allocation_cls, nullptr);
+  EXPECT_EQ(last_isolate_group_data, nullptr);
+
+  // Clear heap sampling callback state.
+  Dart_RegisterHeapSamplingCallback(nullptr);
+}
+
 #endif  // !PRODUCT
 
 }  // namespace dart

runtime/vm/heap/heap_sources.gni

@@ -23,6 +23,8 @@ heap_sources = [
   "pointer_block.h",
   "safepoint.cc",
   "safepoint.h",
+  "sampler.cc",
+  "sampler.h",
   "scavenger.cc",
   "scavenger.h",
   "spaces.h",

runtime/vm/heap/page.h

@@ -248,13 +248,15 @@ class Page {
     owner_ = thread;
     thread->set_top(top_);
     thread->set_end(end_);
+    thread->set_true_end(end_);
   }
   void Release(Thread* thread) {
     ASSERT(owner_ == thread);
     owner_ = nullptr;
     top_ = thread->top();
     thread->set_top(0);
     thread->set_end(0);
+    thread->set_true_end(0);
   }
   void Release() {
     if (owner_ != nullptr) {