Support hot cold splitting in crossgen2

src/coreclr/inc/readytorun.h

@@ -87,6 +87,7 @@ enum class ReadyToRunSectionType : uint32_t
     PgoInstrumentationData      = 117, // Added in V5.2
     ManifestAssemblyMvids       = 118, // Added in V5.3
     CrossModuleInlineInfo       = 119, // Added in V6.2
+    HotColdMap                  = 120, // Added in V8.0
 
     // If you add a new section consider whether it is a breaking or non-breaking change.
     // Usually it is non-breaking, but if it is preferable to have older runtimes fail

src/coreclr/jit/fginline.cpp

@@ -1325,6 +1325,15 @@ void Compiler::fgInsertInlineeBlocks(InlineInfo* pInlineInfo)
 
     lvaGenericsContextInUse |= InlineeCompiler->lvaGenericsContextInUse;
 
+    // If the inlinee compiler encounters switch tables, disable hot/cold splitting in the root compiler.
+    // TODO-CQ: Implement hot/cold splitting of methods with switch tables.
+    if (InlineeCompiler->fgHasSwitch && opts.compProcedureSplitting)
+    {
+        opts.compProcedureSplitting = false;
+        JITDUMP("Turning off procedure splitting for this method, as inlinee compiler encountered switch tables; "
+                "implementation limitation.\n");
+    }
+
 #ifdef FEATURE_SIMD
     if (InlineeCompiler->usesSIMDTypes())
     {

src/coreclr/jit/flowgraph.cpp

@@ -3514,6 +3514,8 @@ PhaseStatus Compiler::fgDetermineFirstColdBlock()
     }
     else
     {
+        bool inFuncletSection = false;
+
         for (lblk = nullptr, block = fgFirstBB; block != nullptr; lblk = block, block = block->bbNext)
         {
             bool blockMustBeInHotSection = false;
@@ -3525,6 +3527,15 @@ PhaseStatus Compiler::fgDetermineFirstColdBlock()
             }
 #endif // HANDLER_ENTRY_MUST_BE_IN_HOT_SECTION
 
+#ifdef FEATURE_EH_FUNCLETS
+            // Make note of if we're in the funclet section,
+            // so we can stop the search early.
+            if (block == fgFirstFuncletBB)
+            {
+                inFuncletSection = true;
+            }
+#endif // FEATURE_EH_FUNCLETS
+
             // Do we have a candidate for the first cold block?
             if (firstColdBlock != nullptr)
             {
@@ -3536,6 +3547,21 @@ PhaseStatus Compiler::fgDetermineFirstColdBlock()
                     // We have to restart the search for the first cold block
                     firstColdBlock       = nullptr;
                     prevToFirstColdBlock = nullptr;
+
+#ifdef FEATURE_EH_FUNCLETS
+                    // If we're already in the funclet section, try to split
+                    // at fgFirstFuncletBB, and stop the search.
+                    if (inFuncletSection)
+                    {
+                        if (fgFuncletsAreCold())
+                        {
+                            firstColdBlock       = fgFirstFuncletBB;
+                            prevToFirstColdBlock = fgFirstFuncletBB->bbPrev;
+                        }
+
+                        break;
+                    }
+#endif // FEATURE_EH_FUNCLETS
                 }
             }
             else // (firstColdBlock == NULL) -- we don't have a candidate for first cold block
@@ -3547,7 +3573,7 @@ PhaseStatus Compiler::fgDetermineFirstColdBlock()
                 // consider splitting at the first funclet; do not consider splitting between funclets,
                 // as this may break unwind info.
                 //
-                if (block == fgFirstFuncletBB)
+                if (inFuncletSection)
                 {
                     if (fgFuncletsAreCold())
                     {

src/coreclr/jit/unwind.cpp

@@ -102,7 +102,15 @@ void Compiler::unwindGetFuncLocations(FuncInfoDsc*             func,
             assert(fgFirstColdBlock != nullptr); // There better be a cold section!
 
             *ppStartLoc = new (this, CMK_UnwindInfo) emitLocation(ehEmitCookie(fgFirstColdBlock));
-            *ppEndLoc   = nullptr; // nullptr end location means the end of the code
+
+            if (fgFirstFuncletBB != nullptr)
+            {
+                *ppEndLoc = new (this, CMK_UnwindInfo) emitLocation(ehEmitCookie(fgFirstFuncletBB));
+            }
+            else
+            {
+                *ppEndLoc = nullptr; // nullptr end location means the end of the code
+            }
         }
     }
     else

src/coreclr/jit/unwindamd64.cpp

@@ -656,17 +656,34 @@ void Compiler::unwindReserve()
 //
 void Compiler::unwindReserveFunc(FuncInfoDsc* func)
 {
-    unwindReserveFuncHelper(func, true);
-
-    if (fgFirstColdBlock != nullptr)
-    {
 #ifdef DEBUG
-        if (!JitConfig.JitFakeProcedureSplitting())
+    // If fake-splitting, treat all unwind info as hot.
+    if (JitConfig.JitFakeProcedureSplitting())
+    {
+        unwindReserveFuncHelper(func, true);
+        return;
+    }
 #endif // DEBUG
+
+    if (func->funKind == FUNC_ROOT)
+    {
+        unwindReserveFuncHelper(func, true);
+
+        // If the function's main body is split, reserve unwind info of size 0 for the cold section.
+        // If only funclets are cold, the main body is hot, so don't make a second call.
+        const bool isFunctionSplit = ((fgFirstColdBlock != nullptr) && (fgFirstColdBlock != fgFirstFuncletBB));
+        if (isFunctionSplit)
         {
             unwindReserveFuncHelper(func, false);
         }
     }
+    else
+    {
+        // Make only one call for funclets.
+        // If function is split and has EH, the funclets will be cold.
+        const bool isFuncletHot = (fgFirstColdBlock == nullptr);
+        unwindReserveFuncHelper(func, isFuncletHot);
+    }
 }
 
 //------------------------------------------------------------------------
@@ -679,8 +696,10 @@ void Compiler::unwindReserveFunc(FuncInfoDsc* func)
 //
 void Compiler::unwindReserveFuncHelper(FuncInfoDsc* func, bool isHotCode)
 {
-    DWORD unwindCodeBytes = 0;
-    if (isHotCode)
+    const bool isFunclet       = (func->funKind != FUNC_ROOT);
+    DWORD      unwindCodeBytes = 0;
+
+    if (isHotCode || isFunclet)
     {
 #ifdef UNIX_AMD64_ABI
         if (generateCFIUnwindCodes())
@@ -717,9 +736,7 @@ void Compiler::unwindReserveFuncHelper(FuncInfoDsc* func, bool isHotCode)
         }
     }
 
-    bool isFunclet  = (func->funKind != FUNC_ROOT);
-    bool isColdCode = !isHotCode;
-
+    const bool isColdCode = !isHotCode;
     eeReserveUnwindInfo(isFunclet, isColdCode, unwindCodeBytes);
 }
 
@@ -779,7 +796,32 @@ void Compiler::unwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode, void* pCo
         {
             endOffset = func->endLoc->CodeOffset(GetEmitter());
         }
+    }
+    else
+    {
+        assert(fgFirstColdBlock != nullptr);
+
+        if (func->coldStartLoc == nullptr)
+        {
+            startOffset = 0;
+        }
+        else
+        {
+            startOffset = func->coldStartLoc->CodeOffset(GetEmitter());
+        }
+
+        if (func->coldEndLoc == nullptr)
+        {
+            endOffset = info.compNativeCodeSize;
+        }
+        else
+        {
+            endOffset = func->coldEndLoc->CodeOffset(GetEmitter());
+        }
+    }
 
+    if (isHotCode || (func->funKind != FUNC_ROOT))
+    {
 #ifdef UNIX_AMD64_ABI
         if (generateCFIUnwindCodes())
         {
@@ -807,28 +849,6 @@ void Compiler::unwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode, void* pCo
             pUnwindBlock = &func->unwindCodes[func->unwindCodeSlot];
         }
     }
-    else
-    {
-        assert(fgFirstColdBlock != nullptr);
-
-        if (func->coldStartLoc == nullptr)
-        {
-            startOffset = 0;
-        }
-        else
-        {
-            startOffset = func->coldStartLoc->CodeOffset(GetEmitter());
-        }
-
-        if (func->coldEndLoc == nullptr)
-        {
-            endOffset = info.compNativeCodeSize;
-        }
-        else
-        {
-            endOffset = func->coldEndLoc->CodeOffset(GetEmitter());
-        }
-    }
 
 #ifdef DEBUG
     if (opts.dspUnwind)
@@ -894,17 +914,34 @@ void Compiler::unwindEmitFunc(FuncInfoDsc* func, void* pHotCode, void* pColdCode
     static_assert_no_msg(FUNC_HANDLER == (FuncKind)CORJIT_FUNC_HANDLER);
     static_assert_no_msg(FUNC_FILTER == (FuncKind)CORJIT_FUNC_FILTER);
 
-    unwindEmitFuncHelper(func, pHotCode, pColdCode, true);
-
-    if (pColdCode != nullptr)
-    {
 #ifdef DEBUG
-        if (!JitConfig.JitFakeProcedureSplitting())
+    // If fake-splitting, treat all unwind info as hot.
+    if (JitConfig.JitFakeProcedureSplitting())
+    {
+        unwindEmitFuncHelper(func, pHotCode, pColdCode, true);
+        return;
+    }
 #endif // DEBUG
+
+    if (func->funKind == FUNC_ROOT)
+    {
+        unwindEmitFuncHelper(func, pHotCode, pColdCode, true);
+
+        // If the function's main body is split, reserve unwind info of size 0 for the cold section.
+        // If only funclets are cold, the main body is hot, so don't make a second call.
+        const bool isFunctionSplit = ((fgFirstColdBlock != nullptr) && (fgFirstColdBlock != fgFirstFuncletBB));
+        if (isFunctionSplit)
         {
             unwindEmitFuncHelper(func, pHotCode, pColdCode, false);
         }
     }
+    else
+    {
+        // Make only one call for funclets.
+        // If function is split and has EH, the funclets will be cold.
+        const bool isFuncletHot = (fgFirstColdBlock == nullptr);
+        unwindEmitFuncHelper(func, pHotCode, pColdCode, isFuncletHot);
+    }
 }
 
 #endif // TARGET_AMD64

src/coreclr/jit/unwindarm.cpp

@@ -572,14 +572,28 @@ void Compiler::unwindReserveFunc(FuncInfoDsc* func)
     }
 #endif // DEBUG
 
+#ifdef FEATURE_EH_FUNCLETS
+    // If hot/cold splitting occurred at fgFirstFuncletBB, then the main body is not split.
+    const bool splitAtFirstFunclet = (funcHasColdSection && (fgFirstColdBlock == fgFirstFuncletBB));
+
+    if (!isFunclet && splitAtFirstFunclet)
+    {
+        funcHasColdSection = false;
+    }
+#endif // FEATURE_EH_FUNCLETS
+
 #if defined(FEATURE_CFI_SUPPORT)
     if (generateCFIUnwindCodes())
     {
+        // Report zero-sized unwind info for cold part of main function
+        // so the EE chains unwind info.
+        // TODO: Support cold EH funclets.
         DWORD unwindCodeBytes = 0;
         if (funcHasColdSection)
         {
             eeReserveUnwindInfo(isFunclet, true /*isColdCode*/, unwindCodeBytes);
         }
+
         unwindCodeBytes = (DWORD)(func->cfiCodes->size() * sizeof(CFI_CODE));
         eeReserveUnwindInfo(isFunclet, false /*isColdCode*/, unwindCodeBytes);
 
@@ -607,7 +621,11 @@ void Compiler::unwindReserveFunc(FuncInfoDsc* func)
     // The ARM Exception Data specification "Function Fragments" section describes this.
     func->uwi.Split();
 
-    func->uwi.Reserve(isFunclet, true);
+    // If the function is split, EH funclets are always cold; skip this call for cold funclets.
+    if (!isFunclet || !funcHasColdSection)
+    {
+        func->uwi.Reserve(isFunclet, true);
+    }
 
     // After the hot section, split and reserve the cold section
 
@@ -644,12 +662,17 @@ void Compiler::unwindEmitFunc(FuncInfoDsc* func, void* pHotCode, void* pColdCode
 #if defined(FEATURE_CFI_SUPPORT)
     if (generateCFIUnwindCodes())
     {
+        // TODO: Support cold EH funclets.
         unwindEmitFuncCFI(func, pHotCode, pColdCode);
         return;
     }
 #endif // FEATURE_CFI_SUPPORT
 
-    func->uwi.Allocate((CorJitFuncKind)func->funKind, pHotCode, pColdCode, true);
+    // If the function is split, EH funclets are always cold; skip this call for cold funclets.
+    if ((func->funKind == FUNC_ROOT) || (func->uwiCold == NULL))
+    {
+        func->uwi.Allocate((CorJitFuncKind)func->funKind, pHotCode, pColdCode, true);
+    }
 
     if (func->uwiCold != NULL)
     {
@@ -1572,8 +1595,6 @@ void UnwindFragmentInfo::Finalize(UNATIVE_OFFSET functionLength)
 
 void UnwindFragmentInfo::Reserve(bool isFunclet, bool isHotCode)
 {
-    assert(isHotCode || !isFunclet); // TODO-CQ: support hot/cold splitting in functions with EH
-
     MergeCodes();
 
     bool isColdCode = !isHotCode;
@@ -1925,7 +1946,6 @@ void UnwindInfo::Split()
 void UnwindInfo::Reserve(bool isFunclet, bool isHotCode)
 {
     assert(uwiInitialized == UWI_INITIALIZED_PATTERN);
-    assert(isHotCode || !isFunclet);
 
     for (UnwindFragmentInfo* pFrag = &uwiFragmentFirst; pFrag != NULL; pFrag = pFrag->ufiNext)
     {

src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs

@@ -69,6 +69,7 @@ public enum ReadyToRunSectionType
         PgoInstrumentationData = 117, // Added in 5.2
         ManifestAssemblyMvids = 118, // Added in 5.3
         CrossModuleInlineInfo = 119, // Added in 6.3
+        HotColdMap = 120, // Added in 8.0
 
         //
         // NativeAOT ReadyToRun sections