Update BDN to latest version with few important bug fixes

NuGet.config

@@ -15,6 +15,7 @@
     <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
     <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
     <add key="benchmark-dotnet-prerelease" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/benchmark-dotnet-prerelease/nuget/v3/index.json" />
+    <add key="bdn-temporary" value="https://ci.appveyor.com/nuget/benchmarkdotnet" />
   </packageSources>
   <disabledPackageSources />
 </configuration>

src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj

@@ -8,8 +8,8 @@
     <_BenchmarkDotNetSourcesN>$([MSBuild]::NormalizeDirectory('$(BenchmarkDotNetSources)'))</_BenchmarkDotNetSourcesN>
   </PropertyGroup>
   <ItemGroup Condition="'$(BenchmarkDotNetSources)' == ''">
-    <PackageReference Include="BenchmarkDotNet" Version="0.13.1.1669" />
-    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.1.1669" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1.1689" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.1.1689" />
   </ItemGroup>
   <ItemGroup Condition="'$(BenchmarkDotNetSources)' != ''">
     <ProjectReference Include="$(_BenchmarkDotNetSourcesN)src\BenchmarkDotNet\BenchmarkDotNet.csproj" SetTargetFramework="TargetFramework=netstandard2.0" />

src/harness/BenchmarkDotNet.Extensions/PartitionFilter.cs

@@ -1,10 +1,6 @@
 using BenchmarkDotNet.Filters;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using BenchmarkDotNet.Running;
 
-
 public class PartitionFilter : IFilter
 {
     private readonly int? _partitionsCount;

src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs

@@ -12,6 +12,7 @@
 using BenchmarkDotNet.Loggers;
 using System.Linq;
 using BenchmarkDotNet.Exporters;
+using System;
 
 namespace BenchmarkDotNet.Extensions
 {
@@ -41,6 +42,7 @@ public static IConfig Create(
             }
 
             var config = ManualConfig.CreateEmpty()
+                .WithBuildTimeout(TimeSpan.FromMinutes(10)) // for slow machines
                 .AddLogger(ConsoleLogger.Default) // log output to console
                 .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) // copy default validators
                 .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) // copy default analysers

src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj

@@ -1,11 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netcoreapp3.1;net5.0</TargetFrameworks>
+    <!-- Used by Python script to narrow down the specified target frameworks to test, and avoid downloading all supported SDKs -->
+    <TargetFrameworks>$(PERFLAB_TARGET_FRAMEWORKS)</TargetFrameworks>
+    <!-- Supported target frameworks -->
+    <TargetFrameworks Condition="'$(TargetFrameworks)' == '' AND '$(OS)' == 'Windows_NT'">net461;netcoreapp3.1;net5.0;net6.0;net7.0</TargetFrameworks>
+    <TargetFrameworks Condition="'$(TargetFrameworks)' == ''">netcoreapp3.1;net5.0;net6.0;net7.0</TargetFrameworks>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\..\benchmarks\micro\MicroBenchmarks.csproj" />
     <ProjectReference Include="..\..\..\harness\BenchmarkDotNet.Extensions\BenchmarkDotNet.Extensions.csproj" />
   </ItemGroup>
 </Project>

src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs

@@ -0,0 +1,133 @@
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.ConsoleArguments;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Parameters;
+using BenchmarkDotNet.Running;
+using MicroBenchmarks;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using Xunit;
+
+namespace BenchmarkDotNet.Extensions.Tests
+{
+    public class PartitionFilterTests
+    {
+        private const int PartitionCount = 30; // same as used by the runtime repo
+
+        [Fact]
+        public void NoBenchmarksAreOmitted_MockData()
+        {
+            Dictionary<BenchmarkCase, int> hits = GenerateBenchmarkCases(4_000).ToDictionary(benchmark => benchmark, _ => 0);
+
+            for (int partitionIndex = 0; partitionIndex < PartitionCount; partitionIndex++)
+            {
+                PartitionFilter filter = new (PartitionCount, partitionIndex);
+
+                foreach (BenchmarkCase benchmark in hits.Keys)
+                {
+                    if (filter.Predicate(benchmark))
+                    {
+                        hits[benchmark] += 1;
+                    }
+                }
+            }
+
+            Assert.All(hits.Values, hitCount => Assert.Equal(1, hitCount));
+        }
+
+        private static IEnumerable<BenchmarkCase> GenerateBenchmarkCases(int count)
+        {
+            MethodInfo publicMethod = typeof(PartitionFilterTests)
+                .GetMethods()
+                .Single(method => method.Name == nameof(NoBenchmarksAreOmitted_MockData));
+
+            Descriptor target = new (typeof(PartitionFilterTests), publicMethod);
+            ParameterInstances parameterInstances = new (Array.Empty<ParameterInstance>());
+            ImmutableConfig config = ManualConfig.CreateEmpty().CreateImmutableConfig();
+
+            for (int i = 0; i < count; i++)
+            {
+                yield return BenchmarkCase.Create(target, Job.Default, parameterInstances, config);
+            }
+        }
+
+        [Fact]
+        public void NoBenchmarksAreOmitted_RealData()
+        {
+            ILogger nullLogger = new NullLogger();
+            IConfig recommendedConfig = RecommendedConfig.Create(
+                artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(PartitionFilterTests).Assembly.Location), "BenchmarkDotNet.Artifacts")),
+                mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty));
+            (bool isSuccess, IConfig parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig);
+            Assert.True(isSuccess);
+
+            Assembly microbenchmarksAssembly = typeof(Categories).Assembly;
+            (bool allTypesValid, IReadOnlyList<Type> runnable) = Running.TypeFilter.GetTypesWithRunnableBenchmarks(
+                Array.Empty<Type>(),
+                new Assembly[1] { microbenchmarksAssembly },
+                nullLogger);
+            Assert.True(allTypesValid);
+
+            BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig, runnable);
+            Dictionary<string, int> idToPartitionIndex = new ();
+
+            for (int i = 0; i < 10; i++)
+            {
+                Dictionary<string, int> hits = allBenchmarks
+                    .SelectMany(benchmark => benchmark.BenchmarksCases)
+                    .ToDictionary(benchmarkCase => GetId(benchmarkCase), _ => 0);
+
+                for (int partitionIndex = 0; partitionIndex < PartitionCount; partitionIndex++)
+                {
+                    PartitionFilter filter = new(PartitionCount, partitionIndex);
+
+                    foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig, runnable).SelectMany(benchmark => benchmark.BenchmarksCases))
+                    {
+                        if (filter.Predicate(benchmark))
+                        {
+                            string id = GetId(benchmark);
+
+                            hits[id] += 1;
+
+                            if (idToPartitionIndex.ContainsKey(id))
+                            {
+                                Assert.Equal(partitionIndex, idToPartitionIndex[id]);
+                            }
+                            else
+                            {
+                                idToPartitionIndex.Add(id, partitionIndex);
+                            }
+                        }
+                    }
+                }
+
+                Assert.All(hits.Values, hitCount => Assert.Equal(1, hitCount));
+            }
+
+            static BenchmarkRunInfo[] GetAllBenchmarks(IConfig parsedConfig, IReadOnlyList<Type> runnable)
+            {
+                return runnable
+                    .Select(type => BenchmarkConverter.TypeToBenchmarks(type, parsedConfig))
+                    .Where(benchmark => benchmark.BenchmarksCases.Any())
+                    .ToArray();
+            }
+
+            static string GetId(BenchmarkCase benchmark) => $"{benchmark.Descriptor.Type.Namespace}{benchmark.DisplayInfo}";
+        }
+
+        private class NullLogger : ILogger
+        {
+            public string Id => string.Empty;
+            public int Priority => default;
+            public void Flush() { }
+            public void Write(LogKind logKind, string text) { }
+            public void WriteLine() { }
+            public void WriteLine(LogKind logKind, string text) { }
+        }
+    }
+}