Merge pull request #51383 from sharwell/faster-segments

Optimize storage for segment information

src/Dependencies/Collections/Internal/SegmentedArrayHelper.cs

@@ -4,6 +4,8 @@
 
 using System;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.CompilerServices;
 
 namespace Microsoft.CodeAnalysis.Collections.Internal
 {
@@ -14,6 +16,52 @@ internal static class SegmentedArrayHelper
         // Large value types may benefit from a smaller number.
         internal const int IntrosortSizeThreshold = 16;
 
+        /// <summary>
+        /// A combination of <see cref="MethodImplOptions.AggressiveInlining"/> and
+        /// <see cref="F:System.Runtime.CompilerServices.MethodImplOptions.AggressiveOptimization"/>.
+        /// </summary>
+        [SuppressMessage("Documentation", "CA1200:Avoid using cref tags with a prefix", Justification = "The field is not supported in all compilation targets.")]
+        internal const MethodImplOptions FastPathMethodImplOptions = MethodImplOptions.AggressiveInlining | (MethodImplOptions)512;
+
+        [MethodImpl(FastPathMethodImplOptions)]
+        internal static int GetSegmentSize<T>()
+        {
+            if (Unsafe.SizeOf<T>() == Unsafe.SizeOf<object>())
+            {
+                return ReferenceTypeSegmentHelper.SegmentSize;
+            }
+            else
+            {
+                return ValueTypeSegmentHelper<T>.SegmentSize;
+            }
+        }
+
+        [MethodImpl(FastPathMethodImplOptions)]
+        internal static int GetSegmentShift<T>()
+        {
+            if (Unsafe.SizeOf<T>() == Unsafe.SizeOf<object>())
+            {
+                return ReferenceTypeSegmentHelper.SegmentShift;
+            }
+            else
+            {
+                return ValueTypeSegmentHelper<T>.SegmentShift;
+            }
+        }
+
+        [MethodImpl(FastPathMethodImplOptions)]
+        internal static int GetOffsetMask<T>()
+        {
+            if (Unsafe.SizeOf<T>() == Unsafe.SizeOf<object>())
+            {
+                return ReferenceTypeSegmentHelper.OffsetMask;
+            }
+            else
+            {
+                return ValueTypeSegmentHelper<T>.OffsetMask;
+            }
+        }
+
         /// <summary>
         /// Calculates the maximum number of elements of size <paramref name="elementSize"/> which can fit into an array
         /// which has the following characteristics:
@@ -24,7 +72,7 @@ internal static class SegmentedArrayHelper
         /// </summary>
         /// <param name="elementSize">The size of the elements in the array.</param>
         /// <returns>The segment size to use for small object heap segmented arrays.</returns>
-        internal static int CalculateSegmentSize(int elementSize)
+        private static int CalculateSegmentSize(int elementSize)
         {
             // Default Large Object Heap size threshold
             // https://github.com/dotnet/runtime/blob/c9d69e38d0e54bea5d188593ef6c3b30139f3ab1/src/coreclr/src/gc/gc.h#L111
@@ -50,7 +98,7 @@ static int ArraySize(int elementSize, int segmentSize)
         /// </summary>
         /// <param name="segmentSize">The number of elements in each page of the segmented array. Must be a power of 2.</param>
         /// <returns>The shift to apply to the absolute index to get the page index within a segmented array.</returns>
-        internal static int CalculateSegmentShift(int segmentSize)
+        private static int CalculateSegmentShift(int segmentSize)
         {
             var segmentShift = 0;
             while (0 != (segmentSize >>= 1))
@@ -67,10 +115,36 @@ internal static int CalculateSegmentShift(int segmentSize)
         /// </summary>
         /// <param name="segmentSize">The number of elements in each page of the segmented array. Must be a power of 2.</param>
         /// <returns>The bit mask to obtain the index within a page from an absolute index within a segmented array.</returns>
-        internal static int CalculateOffsetMask(int segmentSize)
+        private static int CalculateOffsetMask(int segmentSize)
         {
             Debug.Assert(segmentSize == 1 || (segmentSize & (segmentSize - 1)) == 0, "Expected size of 1, or a power of 2");
             return segmentSize - 1;
         }
+
+        internal static class TestAccessor
+        {
+            public static int CalculateSegmentSize(int elementSize)
+                => SegmentedArrayHelper.CalculateSegmentSize(elementSize);
+
+            public static int CalculateSegmentShift(int elementSize)
+                => SegmentedArrayHelper.CalculateSegmentShift(elementSize);
+
+            public static int CalculateOffsetMask(int elementSize)
+                => SegmentedArrayHelper.CalculateOffsetMask(elementSize);
+        }
+
+        private static class ReferenceTypeSegmentHelper
+        {
+            public static readonly int SegmentSize = CalculateSegmentSize(Unsafe.SizeOf<object>());
+            public static readonly int SegmentShift = CalculateSegmentShift(SegmentSize);
+            public static readonly int OffsetMask = CalculateOffsetMask(SegmentSize);
+        }
+
+        private static class ValueTypeSegmentHelper<T>
+        {
+            public static readonly int SegmentSize = CalculateSegmentSize(Unsafe.SizeOf<T>());
+            public static readonly int SegmentShift = CalculateSegmentShift(SegmentSize);
+            public static readonly int OffsetMask = CalculateOffsetMask(SegmentSize);
+        }
     }
 }

src/Dependencies/Collections/SegmentedArray`1.cs

@@ -30,17 +30,38 @@ namespace Microsoft.CodeAnalysis.Collections
         /// including any padding the implementation chooses to add. Specifically, array elements lie <c>sizeof</c>
         /// bytes apart.</para>
         /// </remarks>
-        private static readonly int s_segmentSize = SegmentedArrayHelper.CalculateSegmentSize(Unsafe.SizeOf<T>());
+        private static int SegmentSize
+        {
+            [MethodImpl(SegmentedArrayHelper.FastPathMethodImplOptions)]
+            get
+            {
+                return SegmentedArrayHelper.GetSegmentSize<T>();
+            }
+        }
 
         /// <summary>
         /// The bit shift to apply to an array index to get the page index within <see cref="_items"/>.
         /// </summary>
-        private static readonly int s_segmentShift = SegmentedArrayHelper.CalculateSegmentShift(s_segmentSize);
+        private static int SegmentShift
+        {
+            [MethodImpl(SegmentedArrayHelper.FastPathMethodImplOptions)]
+            get
+            {
+                return SegmentedArrayHelper.GetSegmentShift<T>();
+            }
+        }
 
         /// <summary>
         /// The bit mask to apply to an array index to get the index within a page of <see cref="_items"/>.
         /// </summary>
-        private static readonly int s_offsetMask = SegmentedArrayHelper.CalculateOffsetMask(s_segmentSize);
+        private static int OffsetMask
+        {
+            [MethodImpl(SegmentedArrayHelper.FastPathMethodImplOptions)]
+            get
+            {
+                return SegmentedArrayHelper.GetOffsetMask<T>();
+            }
+        }
 
         private readonly int _length;
         private readonly T[][] _items;
@@ -59,17 +80,17 @@ public SegmentedArray(int length)
             }
             else
             {
-                _items = new T[(length + s_segmentSize - 1) >> s_segmentShift][];
+                _items = new T[(length + SegmentSize - 1) >> SegmentShift][];
                 for (var i = 0; i < _items.Length - 1; i++)
                 {
-                    _items[i] = new T[s_segmentSize];
+                    _items[i] = new T[SegmentSize];
                 }
 
                 // Make sure the last page only contains the number of elements required for the desired length. This
                 // collection is not resizeable so any additional padding would be a waste of space.
                 //
                 // Avoid using (length & s_offsetMask) because it doesn't handle a last page size of s_segmentSize.
-                var lastPageSize = length - ((_items.Length - 1) << s_segmentShift);
+                var lastPageSize = length - ((_items.Length - 1) << SegmentShift);
 
                 _items[_items.Length - 1] = new T[lastPageSize];
                 _length = length;
@@ -94,9 +115,10 @@ private SegmentedArray(int length, T[][] items)
 
         public ref T this[int index]
         {
+            [MethodImpl(SegmentedArrayHelper.FastPathMethodImplOptions)]
             get
             {
-                return ref _items[index >> s_segmentShift][index & s_offsetMask];
+                return ref _items[index >> SegmentShift][index & OffsetMask];
             }
         }
 
@@ -135,7 +157,7 @@ public void CopyTo(Array array, int index)
         {
             for (var i = 0; i < _items.Length; i++)
             {
-                _items[i].CopyTo(array, index + (i * s_segmentSize));
+                _items[i].CopyTo(array, index + (i * SegmentSize));
             }
         }
 
@@ -144,7 +166,7 @@ void ICollection<T>.CopyTo(T[] array, int arrayIndex)
             for (var i = 0; i < _items.Length; i++)
             {
                 ICollection<T> collection = _items[i];
-                collection.CopyTo(array, arrayIndex + (i * s_segmentSize));
+                collection.CopyTo(array, arrayIndex + (i * SegmentSize));
             }
         }
 
@@ -207,7 +229,7 @@ int IList.IndexOf(object? value)
                 var index = list.IndexOf(value);
                 if (index >= 0)
                 {
-                    return index + i * s_segmentSize;
+                    return index + i * SegmentSize;
                 }
             }
 
@@ -222,7 +244,7 @@ int IList<T>.IndexOf(T value)
                 var index = list.IndexOf(value);
                 if (index >= 0)
                 {
-                    return index + i * s_segmentSize;
+                    return index + i * SegmentSize;
                 }
             }
 
@@ -393,7 +415,7 @@ public TestAccessor(SegmentedArray<T> array)
                 _array = array;
             }
 
-            public static int SegmentSize => s_segmentSize;
+            public static int SegmentSize => SegmentedArray<T>.SegmentSize;
 
             public T[][] Items => _array._items;
         }

src/Tools/IdeCoreBenchmarks/IdeCoreBenchmarks.csproj

@@ -5,7 +5,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFrameworks>netcoreapp3.1;net472</TargetFrameworks>
+    <TargetFrameworks>net5.0;netcoreapp3.1;net472</TargetFrameworks>
     <IsShipping>false</IsShipping>
     <PlatformTarget>AnyCPU</PlatformTarget>
     <!-- Automatically generate the necessary assembly binding redirects -->

src/Tools/IdeCoreBenchmarks/SegmentedArrayBenchmarks.cs

@@ -0,0 +1,75 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Attributes;
+using Microsoft.CodeAnalysis.Collections;
+
+namespace IdeCoreBenchmarks
+{
+    [DisassemblyDiagnoser]
+    public class SegmentedArrayBenchmarks
+    {
+        private int[] _values = null!;
+        private object?[] _valuesObject = null!;
+
+        private SegmentedArray<int> _segmentedValues;
+        private SegmentedArray<object?> _segmentedValuesObject;
+
+        [Params(100000)]
+        public int Count { get; set; }
+
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+            _values = new int[Count];
+            _valuesObject = new object?[Count];
+            _segmentedValues = new SegmentedArray<int>(Count);
+            _segmentedValuesObject = new SegmentedArray<object?>(Count);
+        }
+
+        [Benchmark(Description = "int[]", Baseline = true)]
+        public void ShiftAllArray()
+        {
+            for (var i = 0; i < _values.Length - 1; i++)
+            {
+                _values[i] = _values[i + 1];
+            }
+
+            _values[^1] = 0;
+        }
+
+        [Benchmark(Description = "object[]")]
+        public void ShiftAllArrayObject()
+        {
+            for (var i = 0; i < _valuesObject.Length - 1; i++)
+            {
+                _valuesObject[i] = _valuesObject[i + 1];
+            }
+
+            _valuesObject[^1] = null;
+        }
+
+        [Benchmark(Description = "SegmentedArray<int>")]
+        public void ShiftAllSegmented()
+        {
+            for (var i = 0; i < _segmentedValues.Length - 1; i++)
+            {
+                _segmentedValues[i] = _segmentedValues[i + 1];
+            }
+
+            _segmentedValues[^1] = 0;
+        }
+
+        [Benchmark(Description = "SegmentedArray<object>")]
+        public void ShiftAllSegmentedObject()
+        {
+            for (var i = 0; i < _segmentedValuesObject.Length - 1; i++)
+            {
+                _segmentedValuesObject[i] = _segmentedValuesObject[i + 1];
+            }
+
+            _segmentedValuesObject[^1] = null;
+        }
+    }
+}

src/Workspaces/CoreTest/Collections/SegmentedArrayHelperTests.cs

@@ -44,7 +44,7 @@ public void CalculateSegmentSize(int elementSize)
                 _ => throw ExceptionUtilities.Unreachable,
             };
 
-            Assert.Equal(expected, SegmentedArrayHelper.CalculateSegmentSize(elementSize));
+            Assert.Equal(expected, SegmentedArrayHelper.TestAccessor.CalculateSegmentSize(elementSize));
         }
 
         [Theory]
@@ -69,7 +69,7 @@ public void CalculateSegmentShift(int segmentSize)
                 _ => throw ExceptionUtilities.Unreachable,
             };
 
-            Assert.Equal(expected, SegmentedArrayHelper.CalculateSegmentShift(segmentSize));
+            Assert.Equal(expected, SegmentedArrayHelper.TestAccessor.CalculateSegmentShift(segmentSize));
         }
 
         [Theory]
@@ -94,7 +94,7 @@ public void CalculateOffsetMask(int segmentSize)
                 _ => throw ExceptionUtilities.Unreachable,
             };
 
-            Assert.Equal(expected, SegmentedArrayHelper.CalculateOffsetMask(segmentSize));
+            Assert.Equal(expected, SegmentedArrayHelper.TestAccessor.CalculateOffsetMask(segmentSize));
         }
     }
 }