ReadyToRun benchmarks

[jairbubbles]

Hello,

I'm using BenchmarkDotNet to measure the boostrap time of my application and wanted to see the impact of using ReadyToRun assemblies (see https://learn.microsoft.com/en-us/dotnet/core/deploying/ready-to-run).
It took me some time to figure it out so I'm sharing my recipe hoping it can be helpful to others.

You need to publish the application in order to test ReadyToRun assemblies so a custom toolchain is needed:

using BenchmarkDotNet.Characteristics;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Running;
using BenchmarkDotNet.Toolchains;
using BenchmarkDotNet.Toolchains.CsProj;
using BenchmarkDotNet.Toolchains.DotNetCli;
using BenchmarkDotNet.Toolchains.Results;
using BenchmarkDotNet.Validators;

public class ReadyToRunToolchain : Toolchain, IEquatable<ReadyToRunToolchain>
{
    public static readonly IToolchain NetCoreApp20 = From(NetCoreAppSettings.NetCoreApp20);
    public static readonly IToolchain NetCoreApp21 = From(NetCoreAppSettings.NetCoreApp21);
    public static readonly IToolchain NetCoreApp22 = From(NetCoreAppSettings.NetCoreApp22);
    public static readonly IToolchain NetCoreApp30 = From(NetCoreAppSettings.NetCoreApp30);
    public static readonly IToolchain NetCoreApp31 = From(NetCoreAppSettings.NetCoreApp31);
    public static readonly IToolchain NetCoreApp50 = From(NetCoreAppSettings.NetCoreApp50);
    public static readonly IToolchain NetCoreApp60 = From(NetCoreAppSettings.NetCoreApp60);
    public static readonly IToolchain NetCoreApp70 = From(NetCoreAppSettings.NetCoreApp70);
    public static readonly IToolchain NetCoreApp80 = From(NetCoreAppSettings.NetCoreApp80);
    public static readonly IToolchain NetCoreApp90 = From(NetCoreAppSettings.NetCoreApp90);

    internal ReadyToRunToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor)
        : base(name, generator, builder, executor)
    {
    }

    public static IToolchain From(NetCoreAppSettings settings)
        => new ReadyToRunToolchain(settings.Name + " (ReadyToRun)",
            new CsProjGenerator(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, settings.RuntimeFrameworkVersion),
            new ReadyToRunBuilder(settings.CustomDotNetCliPath),
            new DotNetCliExecutor(settings.CustomDotNetCliPath));

    public override IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCase, IResolver resolver)
    {
        foreach (var validationError in base.Validate(benchmarkCase, resolver))
        {
            yield return validationError;
        }

        if (benchmarkCase.Job.HasValue(EnvironmentMode.JitCharacteristic) && benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.LegacyJit)
        {
            yield return new ValidationError(true,
                $"Currently dotnet cli toolchain supports only RyuJit, benchmark '{benchmarkCase.DisplayInfo}' will not be executed",
                benchmarkCase);
        }
        if (benchmarkCase.Job.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver))
        {
            yield return new ValidationError(true,
                $"Currently project.json does not support CpuGroups (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed",
                benchmarkCase);
        }
        if (benchmarkCase.Job.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver))
        {
            yield return new ValidationError(true,
                $"Currently project.json does not support gcAllowVeryLargeObjects (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed",
                benchmarkCase);
        }
    }

    public override bool Equals(object obj) => obj is ReadyToRunToolchain typed && Equals(typed);

    public bool Equals(ReadyToRunToolchain other) => Generator.Equals(other.Generator);

    public override int GetHashCode() => Generator.GetHashCode();

    public class ReadyToRunBuilder : IBuilder
    {
        public ReadyToRunBuilder(
            string customDotNetCliPath = null,
            IReadOnlyList<EnvironmentVariable> environmentVariables = null)
        {
            CustomDotNetCliPath = customDotNetCliPath;
            EnvironmentVariables = environmentVariables;
        }

        private string CustomDotNetCliPath { get; }

        private IReadOnlyList<EnvironmentVariable> EnvironmentVariables { get; }

        public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
            => new DotNetCliCommand(
                    CustomDotNetCliPath,
                    "-r win-x64 -p:PublishReadyToRun=true",
                    generateResult,
                    logger,
                    buildPartition,
                    EnvironmentVariables,
                    buildPartition.Timeout)
                .Publish().ToBuildResult(generateResult);
    }
}

In the .csproj, it's also important to match the same target platform:

  <PropertyGroup>
    <PlatformTarget>x64</PlatformTarget>
  </PropertyGroup>

Then you can use the toolchain in your program.cs:

BenchmarkRunner.Run<BootstrapBenchmarks>(DefaultConfig.Instance
    .AddJob(Job.Default
        .WithId(".NET 7.0 (R2R)")
        .WithStrategy(RunStrategy.ColdStart)
        .WithRuntime(CoreRuntime.Core70)
        .WithToolchain(ReadyToRunToolchain.NetCoreApp70)
        .WithLaunchCount(10)
        .WithIterationCount(1)
        .WithInvocationCount(1)
        .WithWarmupCount(0)));