Generate benchmarking report

.github/workflows/bvt-appleclang.yml

@@ -9,7 +9,7 @@ jobs:
   bvt-appleclang:
     runs-on: macos-15
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ inputs.branch }}
 
@@ -26,4 +26,11 @@ jobs:
 
     - name: run benchmarks
       run: |
-        ./build/benchmarks/msft_proxy_benchmarks --benchmark_repetitions=100 --benchmark_report_aggregates_only=true --benchmark_min_time=0.1s --benchmark_enable_random_interleaving=true
+        cd build/benchmarks
+        ./msft_proxy_benchmarks --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_enable_random_interleaving=true --benchmark_out=benchmarking-results.json
+
+    - name: archive benchmarking results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmarking-results-appleclang
+        path: build/benchmarks/benchmarking-results.json

.github/workflows/bvt-clang.yml

@@ -9,7 +9,7 @@ jobs:
   bvt-clang:
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ inputs.branch }}
 
@@ -50,4 +50,11 @@ jobs:
 
     - name: run benchmarks
       run: |
-        ./build-clang-18/benchmarks/msft_proxy_benchmarks --benchmark_repetitions=100 --benchmark_report_aggregates_only=true --benchmark_min_time=0.1s --benchmark_enable_random_interleaving=true
+        cd build-clang-18/benchmarks
+        ./msft_proxy_benchmarks --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_enable_random_interleaving=true --benchmark_out=benchmarking-results.json
+
+    - name: archive benchmarking results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmarking-results-clang
+        path: build-clang-18/benchmarks/benchmarking-results.json

.github/workflows/bvt-gcc.yml

@@ -9,7 +9,7 @@ jobs:
   bvt-gcc:
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ inputs.branch }}
 
@@ -50,4 +50,11 @@ jobs:
 
     - name: run benchmarks
       run: |
-        ./build-gcc-14/benchmarks/msft_proxy_benchmarks --benchmark_repetitions=100 --benchmark_report_aggregates_only=true --benchmark_min_time=0.1s --benchmark_enable_random_interleaving=true
+        cd build-gcc-14/benchmarks
+        ./msft_proxy_benchmarks --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_enable_random_interleaving=true --benchmark_out=benchmarking-results.json
+
+    - name: archive benchmarking results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmarking-results-gcc
+        path: build-gcc-14/benchmarks/benchmarking-results.json

.github/workflows/bvt-msvc.yml

@@ -9,7 +9,7 @@ jobs:
   bvt-msvc:
     runs-on: windows-2022
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ inputs.branch }}
 
@@ -21,4 +21,11 @@ jobs:
 
     - name: run benchmarks
       run: |
-        .\build\benchmarks\Release\msft_proxy_benchmarks.exe --benchmark_repetitions=100 --benchmark_report_aggregates_only=true --benchmark_min_time=0.1s --benchmark_enable_random_interleaving=true
+        cd build\benchmarks
+        .\Release\msft_proxy_benchmarks.exe --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_enable_random_interleaving=true --benchmark_out=benchmarking-results.json
+
+    - name: archive benchmarking results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmarking-results-msvc
+        path: build/benchmarks/benchmarking-results.json

.github/workflows/bvt-report.yml

@@ -0,0 +1,35 @@
+on:
+  workflow_call:
+    inputs:
+      branch:
+        type: string
+        required: false
+
+jobs:
+  bvt-report:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.branch }}
+
+    - name: download all workflow run artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
+
+    - name: build report generator
+      run: |
+        cd tools/report_generator
+        cmake -B build -DCMAKE_BUILD_TYPE=Release
+        cmake --build build -j
+
+    - name: generate report
+      run: |
+        tools/report_generator/build/report_generator tools/report_generator/report-config.json ${{ github.sha }} artifacts benchmarking-report.md
+
+    - name: archive benchmarking report
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmarking-report
+        path: benchmarking-report.md

.github/workflows/pipeline-ci.yml

@@ -6,10 +6,6 @@ on:
   pull_request:
     branches: [ main, release/** ]
 
-env:
-  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
-  BUILD_TYPE: Release
-
 jobs:
   run-bvt-gcc:
     uses: ./.github/workflows/bvt-gcc.yml
@@ -26,3 +22,8 @@ jobs:
   run-bvt-appleclang:
     uses: ./.github/workflows/bvt-appleclang.yml
     name: Run BVT with AppleClang
+
+  report:
+    uses: ./.github/workflows/bvt-report.yml
+    name: Generate report
+    needs: [run-bvt-gcc, run-bvt-clang, run-bvt-msvc, run-bvt-appleclang]

.github/workflows/pipeline-release.yml

@@ -65,6 +65,11 @@ jobs:
     with:
       branch: release/${{ github.event.inputs.version }}
 
+  report:
+    uses: ./.github/workflows/bvt-report.yml
+    name: Generate report
+    needs: [run-bvt-gcc, run-bvt-clang, run-bvt-msvc, run-bvt-appleclang]
+
   draft-release:
     runs-on: windows-latest
     needs: [run-bvt-gcc, run-bvt-clang, run-bvt-msvc, run-bvt-appleclang]

benchmarks/proxy_management_benchmark.cpp

@@ -87,7 +87,7 @@ void BM_SmallObjectManagementWithSharedPtr(benchmark::State& state) {
 }
 
 void BM_SmallObjectManagementWithSharedPtr_Pooled(benchmark::State& state) {
-  std::pmr::unsynchronized_pool_resource pool;
+  static std::pmr::unsynchronized_pool_resource pool;
   std::pmr::polymorphic_allocator<> alloc{&pool};
   for (auto _ : state) {
     std::vector<std::shared_ptr<void>> data;
@@ -128,7 +128,7 @@ void BM_LargeObjectManagementWithProxy(benchmark::State& state) {
 }
 
 void BM_LargeObjectManagementWithProxy_Pooled(benchmark::State& state) {
-  std::pmr::unsynchronized_pool_resource pool;
+  static std::pmr::unsynchronized_pool_resource pool;
   std::pmr::polymorphic_allocator<> alloc{&pool};
   for (auto _ : state) {
     std::vector<pro::proxy<DefaultFacade>> data;
@@ -169,7 +169,7 @@ void BM_LargeObjectManagementWithSharedPtr(benchmark::State& state) {
 }
 
 void BM_LargeObjectManagementWithSharedPtr_Pooled(benchmark::State& state) {
-  std::pmr::unsynchronized_pool_resource pool;
+  static std::pmr::unsynchronized_pool_resource pool;
   std::pmr::polymorphic_allocator<> alloc{&pool};
   for (auto _ : state) {
     std::vector<std::shared_ptr<void>> data;

tools/report_generator/CMakeLists.txt

@@ -0,0 +1,28 @@
+cmake_minimum_required(VERSION 3.5)
+
+project(report_generator)
+
+include(FetchContent)
+# The policy uses the download time for timestamp, instead of the timestamp in the archive. This
+# allows for proper rebuilds when a projects URL changes.
+if(POLICY CMP0135)
+  cmake_policy(SET CMP0135 NEW)
+endif()
+
+FetchContent_Declare(
+  nlohmann_json
+  URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+  URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
+)
+FetchContent_MakeAvailable(nlohmann_json)
+
+add_executable(report_generator main.cpp)
+
+target_link_libraries(report_generator PRIVATE nlohmann_json::nlohmann_json)
+
+target_compile_features(report_generator PRIVATE cxx_std_20)
+if (MSVC)
+  target_compile_options(report_generator PRIVATE /W4 /WX)
+else()
+  target_compile_options(report_generator PRIVATE -Wall -Wextra -Wpedantic -Werror)
+endif()

tools/report_generator/main.cpp

@@ -0,0 +1,176 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+#include <cstdio>
+#include <chrono>
+#include <filesystem>
+#include <format>
+#include <fstream>
+#include <iostream>
+#include <string_view>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+
+struct EnvironmentInfo {
+  std::string Name;
+  std::string Description;
+
+  friend void to_json(nlohmann::json& j, const EnvironmentInfo& e) {
+    j = nlohmann::json{
+        {"Name", e.Name},
+        {"Description", e.Description}
+    };
+  }
+
+  friend void from_json(const nlohmann::json& j, EnvironmentInfo& e) {
+    j.at("Name").get_to(e.Name);
+    j.at("Description").get_to(e.Description);
+  }
+};
+
+struct MetricInfo {
+  std::string Name;
+  std::string TargetBenchmarkName;
+  std::string BaselineBenchmarkName;
+
+  friend void to_json(nlohmann::json& j, const MetricInfo& m) {
+    j = nlohmann::json{
+        {"Name", m.Name},
+        {"TargetBenchmarkName", m.TargetBenchmarkName},
+        {"BaselineBenchmarkName", m.BaselineBenchmarkName}
+    };
+  }
+
+  friend void from_json(const nlohmann::json& j, MetricInfo& m) {
+    j.at("Name").get_to(m.Name);
+    j.at("TargetBenchmarkName").get_to(m.TargetBenchmarkName);
+    j.at("BaselineBenchmarkName").get_to(m.BaselineBenchmarkName);
+  }
+};
+
+struct ReportConfig {
+  std::string TargetName;
+  double YellowIndicatorThreshold;
+  std::vector<EnvironmentInfo> Environments;
+  std::vector<MetricInfo> Metrics;
+
+  friend void to_json(nlohmann::json& j, const ReportConfig& rc) {
+    j = nlohmann::json{
+        {"TargetName", rc.TargetName},
+        {"YellowIndicatorThreshold", rc.YellowIndicatorThreshold},
+        {"Environments", rc.Environments},
+        {"Metrics", rc.Metrics}
+    };
+  }
+
+  friend void from_json(const nlohmann::json& j, ReportConfig& rc) {
+    j.at("TargetName").get_to(rc.TargetName);
+    j.at("YellowIndicatorThreshold").get_to(rc.YellowIndicatorThreshold);
+    j.at("Environments").get_to(rc.Environments);
+    j.at("Metrics").get_to(rc.Metrics);
+  }
+};
+
+const std::string_view MedianSuffix = "_median";
+
+std::unordered_map<std::string, double> Parse(const std::filesystem::path& file) {
+  nlohmann::json obj;
+  {
+    std::ifstream in;
+    in.exceptions(std::ios_base::failbit | std::ios_base::badbit);
+    in.open(file, std::ios_base::in | std::ios_base::binary);
+    in >> obj;
+  }
+  std::unordered_map<std::string, double> result;
+  for (auto& node : obj["benchmarks"]) {
+    std::string name = node["name"];
+    if (name.ends_with(MedianSuffix)) {
+      name.resize(name.size() - MedianSuffix.size());
+      double value = node["real_time"];
+      result.emplace(std::move(name), value);
+    }
+  }
+  return result;
+}
+
+void GenerateReport(const std::filesystem::path& config_path, const std::string& commit_id, const std::filesystem::path& source, const std::filesystem::path& output) {
+  ReportConfig config;
+  {
+    nlohmann::json obj;
+    std::ifstream in;
+    in.exceptions(std::ios_base::failbit | std::ios_base::badbit);
+    in.open(config_path, std::ios_base::in | std::ios_base::binary);
+    in >> obj;
+    obj.get_to(config);
+  }
+  std::vector<std::unordered_map<std::string, double>> benchmarks;
+  benchmarks.reserve(config.Environments.size());
+  for (auto& environment : config.Environments) {
+    benchmarks.push_back(Parse(source / std::format("benchmarking-results-{}", environment.Name) / "benchmarking-results.json"));
+  }
+  std::ofstream out;
+  out.exceptions(std::ios_base::failbit | std::ios_base::badbit);
+  out.open(output, std::ios_base::out | std::ios_base::trunc | std::ios_base::binary);
+  out << "## Benchmarking Report\n";
+  out << "\n";
+  out << "- Generated for: [Microsoft \"Proxy\" library](https://github.com/microsoft/proxy)\n";
+  out << "- Commit ID: [" << commit_id << "](https://github.com/microsoft/proxy/commit/" << commit_id << ")\n";
+  out << "- Generated at: " << std::format("{:%FT%TZ}", std::chrono::utc_clock::now()) << "\n";
+  out << "\n";
+  out << "| |";
+  for (auto& environment : config.Environments) {
+    out << " " << environment.Description << " |";
+  }
+  out << "\n";
+  out << "| - |";
+  for (std::size_t i = 0; i < config.Environments.size(); ++i) {
+    out << " - |";
+  }
+  out << "\n";
+  for (auto& metric : config.Metrics) {
+    out << "| " << metric.Name << " |";
+    for (auto& benchmark : benchmarks) {
+      double target = benchmark.at(metric.TargetBenchmarkName);
+      double baseline = benchmark.at(metric.BaselineBenchmarkName);
+      double rate = (baseline - target) * 100 / target;
+      bool is_negative = rate < 0;
+      if (is_negative) {
+        rate = -rate;
+      }
+      out << " ";
+      if (rate < config.YellowIndicatorThreshold) {
+        out << "\xf0\x9f\x9f\xa1";  // Yellow circle
+      } else if (is_negative) {
+        out << "\xf0\x9f\x94\xb4";  // Red circle
+      } else {
+        out << "\xf0\x9f\x9f\xa2";  // Green circle
+      }
+      auto rate_str = std::format("{:.1f}", rate);
+      std::string message;
+      if (rate_str == "0.0") {
+        out << config.TargetName << " has similar performance";
+      } else {
+        out << config.TargetName << " is about **" << rate_str << "% " << (is_negative ? "slower" : "faster") << "**";
+      }
+      out << " |";
+    }
+    out << "\n";
+  }
+}
+
+int main(int argc, char** argv) {
+  if (argc != 5) {
+    puts("Usage: report_generator <config file path> <commit ID> <benchmarking results directory> <output file path>");
+    return 0;
+  }
+  try {
+    GenerateReport(argv[1], argv[2], argv[3], argv[4]);
+  } catch (const std::exception& e) {
+    fprintf(stderr, "An error occurred: %s\n", e.what());
+    return 1;
+  }
+  return 0;
+}