From a2261ad66400c3145f96ebff0d9b75e910fa89dd Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <nnethercote@mozilla.com>
Date: Fri, 13 Sep 2019 10:46:05 +1000
Subject: [PATCH] Inline `mark_neighbours_as_waiting_from`.

This function is very hot, doesn't get inlined because it's recursive,
and the function calls are significant.

This commit splits it into inlined and uninlined variants, and uses the
inlined variant for the hot call site. This wins several percent on a
few benchmarks.
---
 .../obligation_forest/mod.rs                    | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/librustc_data_structures/obligation_forest/mod.rs b/src/librustc_data_structures/obligation_forest/mod.rs
index 04d2b23ab1a1c..6c52e626ababd 100644
--- a/src/librustc_data_structures/obligation_forest/mod.rs
+++ b/src/librustc_data_structures/obligation_forest/mod.rs
@@ -559,13 +559,20 @@ impl<O: ForestObligation> ObligationForest<O> {
         trace
     }
 
-    #[inline]
-    fn mark_neighbors_as_waiting_from(&self, node: &Node<O>) {
+    // This always-inlined function is for the hot call site.
+    #[inline(always)]
+    fn inlined_mark_neighbors_as_waiting_from(&self, node: &Node<O>) {
         for dependent in node.parent.iter().chain(node.dependents.iter()) {
             self.mark_as_waiting_from(&self.nodes[dependent.get()]);
         }
     }
 
+    // This never-inlined function is for the cold call site.
+    #[inline(never)]
+    fn uninlined_mark_neighbors_as_waiting_from(&self, node: &Node<O>) {
+        self.inlined_mark_neighbors_as_waiting_from(node)
+    }
+
     /// Marks all nodes that depend on a pending node as `NodeState::Waiting`.
     fn mark_as_waiting(&self) {
         for node in &self.nodes {
@@ -576,7 +583,8 @@ impl<O: ForestObligation> ObligationForest<O> {
 
         for node in &self.nodes {
             if node.state.get() == NodeState::Pending {
-                self.mark_neighbors_as_waiting_from(node);
+                // This call site is hot.
+                self.inlined_mark_neighbors_as_waiting_from(node);
             }
         }
     }
@@ -588,7 +596,8 @@ impl<O: ForestObligation> ObligationForest<O> {
             NodeState::Pending | NodeState::Done => {},
         }
 
-        self.mark_neighbors_as_waiting_from(node);
+        // This call site is cold.
+        self.uninlined_mark_neighbors_as_waiting_from(node);
     }
 
     /// Compresses the vector, removing all popped nodes. This adjusts