diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index 0463f5d3a49..fe062cce519 100644 --- a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -8,8 +8,8 @@ <_BenchmarkDotNetSourcesN>$([MSBuild]::NormalizeDirectory('$(BenchmarkDotNetSources)')) - - + + diff --git a/src/harness/BenchmarkDotNet.Extensions/PartitionFilter.cs b/src/harness/BenchmarkDotNet.Extensions/PartitionFilter.cs index 16ae22f3167..8fa516618f4 100644 --- a/src/harness/BenchmarkDotNet.Extensions/PartitionFilter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/PartitionFilter.cs @@ -1,10 +1,6 @@ using BenchmarkDotNet.Filters; -using System; -using System.Collections.Generic; -using System.Linq; using BenchmarkDotNet.Running; - public class PartitionFilter : IFilter { private readonly int? _partitionsCount; diff --git a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs index ba00c76d01f..d6ebd6f948a 100644 --- a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs +++ b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs @@ -12,6 +12,7 @@ using BenchmarkDotNet.Loggers; using System.Linq; using BenchmarkDotNet.Exporters; +using System; namespace BenchmarkDotNet.Extensions { @@ -41,6 +42,7 @@ public static IConfig Create( } var config = ManualConfig.CreateEmpty() + .WithBuildTimeout(TimeSpan.FromMinutes(10)) // for slow machines .AddLogger(ConsoleLogger.Default) // log output to console .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) // copy default validators .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) // copy default analysers diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj index cb8c8f38210..276f3bcb072 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj @@ -1,11 +1,16 @@  - netcoreapp3.1;net5.0 + + $(PERFLAB_TARGET_FRAMEWORKS) + + net461;netcoreapp3.1;net5.0;net6.0;net7.0 + netcoreapp3.1;net5.0;net6.0;net7.0 false + diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs new file mode 100644 index 00000000000..7fb156d2648 --- /dev/null +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs @@ -0,0 +1,133 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Parameters; +using BenchmarkDotNet.Running; +using MicroBenchmarks; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Reflection; +using Xunit; + +namespace BenchmarkDotNet.Extensions.Tests +{ + public class PartitionFilterTests + { + private const int PartitionCount = 30; // same as used by the runtime repo + + [Fact] + public void NoBenchmarksAreOmitted_MockData() + { + Dictionary hits = GenerateBenchmarkCases(4_000).ToDictionary(benchmark => benchmark, _ => 0); + + for (int partitionIndex = 0; partitionIndex < PartitionCount; partitionIndex++) + { + PartitionFilter filter = new (PartitionCount, partitionIndex); + + foreach (BenchmarkCase benchmark in hits.Keys) + { + if (filter.Predicate(benchmark)) + { + hits[benchmark] += 1; + } + } + } + + Assert.All(hits.Values, hitCount => Assert.Equal(1, hitCount)); + } + + private static IEnumerable GenerateBenchmarkCases(int count) + { + MethodInfo publicMethod = typeof(PartitionFilterTests) + .GetMethods() + .Single(method => method.Name == nameof(NoBenchmarksAreOmitted_MockData)); + + Descriptor target = new (typeof(PartitionFilterTests), publicMethod); + ParameterInstances parameterInstances = new (Array.Empty()); + ImmutableConfig config = ManualConfig.CreateEmpty().CreateImmutableConfig(); + + for (int i = 0; i < count; i++) + { + yield return BenchmarkCase.Create(target, Job.Default, parameterInstances, config); + } + } + + [Fact] + public void NoBenchmarksAreOmitted_RealData() + { + ILogger nullLogger = new NullLogger(); + IConfig recommendedConfig = RecommendedConfig.Create( + artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(PartitionFilterTests).Assembly.Location), "BenchmarkDotNet.Artifacts")), + mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty)); + (bool isSuccess, IConfig parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig); + Assert.True(isSuccess); + + Assembly microbenchmarksAssembly = typeof(Categories).Assembly; + (bool allTypesValid, IReadOnlyList runnable) = Running.TypeFilter.GetTypesWithRunnableBenchmarks( + Array.Empty(), + new Assembly[1] { microbenchmarksAssembly }, + nullLogger); + Assert.True(allTypesValid); + + BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig, runnable); + Dictionary idToPartitionIndex = new (); + + for (int i = 0; i < 10; i++) + { + Dictionary hits = allBenchmarks + .SelectMany(benchmark => benchmark.BenchmarksCases) + .ToDictionary(benchmarkCase => GetId(benchmarkCase), _ => 0); + + for (int partitionIndex = 0; partitionIndex < PartitionCount; partitionIndex++) + { + PartitionFilter filter = new(PartitionCount, partitionIndex); + + foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig, runnable).SelectMany(benchmark => benchmark.BenchmarksCases)) + { + if (filter.Predicate(benchmark)) + { + string id = GetId(benchmark); + + hits[id] += 1; + + if (idToPartitionIndex.ContainsKey(id)) + { + Assert.Equal(partitionIndex, idToPartitionIndex[id]); + } + else + { + idToPartitionIndex.Add(id, partitionIndex); + } + } + } + } + + Assert.All(hits.Values, hitCount => Assert.Equal(1, hitCount)); + } + + static BenchmarkRunInfo[] GetAllBenchmarks(IConfig parsedConfig, IReadOnlyList runnable) + { + return runnable + .Select(type => BenchmarkConverter.TypeToBenchmarks(type, parsedConfig)) + .Where(benchmark => benchmark.BenchmarksCases.Any()) + .ToArray(); + } + + static string GetId(BenchmarkCase benchmark) => $"{benchmark.Descriptor.Type.Namespace}{benchmark.DisplayInfo}"; + } + + private class NullLogger : ILogger + { + public string Id => string.Empty; + public int Priority => default; + public void Flush() { } + public void Write(LogKind logKind, string text) { } + public void WriteLine() { } + public void WriteLine(LogKind logKind, string text) { } + } + } +}