From b71dab47cce00de4414d6e5c9baeb8552b2b23ed Mon Sep 17 00:00:00 2001 From: Mingxin Wang Date: Tue, 22 Oct 2024 14:16:57 +0800 Subject: [PATCH] Generate benchmarking report (#181) --- .github/workflows/bvt-appleclang.yml | 11 +- .github/workflows/bvt-clang.yml | 11 +- .github/workflows/bvt-gcc.yml | 11 +- .github/workflows/bvt-msvc.yml | 11 +- .github/workflows/bvt-report.yml | 35 +++++ .github/workflows/pipeline-ci.yml | 9 +- .github/workflows/pipeline-release.yml | 5 + benchmarks/proxy_management_benchmark.cpp | 6 +- tools/report_generator/CMakeLists.txt | 28 ++++ tools/report_generator/main.cpp | 176 ++++++++++++++++++++++ tools/report_generator/report-config.json | 84 +++++++++++ 11 files changed, 372 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/bvt-report.yml create mode 100644 tools/report_generator/CMakeLists.txt create mode 100644 tools/report_generator/main.cpp create mode 100644 tools/report_generator/report-config.json diff --git a/.github/workflows/bvt-appleclang.yml b/.github/workflows/bvt-appleclang.yml index d881ca4..3f8af46 100644 --- a/.github/workflows/bvt-appleclang.yml +++ b/.github/workflows/bvt-appleclang.yml @@ -9,7 +9,7 @@ jobs: bvt-appleclang: runs-on: macos-15 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} @@ -26,4 +26,11 @@ jobs: - name: run benchmarks run: | - ./build/benchmarks/msft_proxy_benchmarks --benchmark_repetitions=100 --benchmark_report_aggregates_only=true --benchmark_min_time=0.1s --benchmark_enable_random_interleaving=true + cd build/benchmarks + ./msft_proxy_benchmarks --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_enable_random_interleaving=true --benchmark_out=benchmarking-results.json + + - name: archive benchmarking results + uses: actions/upload-artifact@v4 + with: + name: benchmarking-results-appleclang + path: build/benchmarks/benchmarking-results.json diff --git a/.github/workflows/bvt-clang.yml b/.github/workflows/bvt-clang.yml index c45eb01..12893cd 100644 --- a/.github/workflows/bvt-clang.yml +++ b/.github/workflows/bvt-clang.yml @@ -9,7 +9,7 @@ jobs: bvt-clang: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} @@ -50,4 +50,11 @@ jobs: - name: run benchmarks run: | - ./build-clang-18/benchmarks/msft_proxy_benchmarks --benchmark_repetitions=100 --benchmark_report_aggregates_only=true --benchmark_min_time=0.1s --benchmark_enable_random_interleaving=true + cd build-clang-18/benchmarks + ./msft_proxy_benchmarks --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_enable_random_interleaving=true --benchmark_out=benchmarking-results.json + + - name: archive benchmarking results + uses: actions/upload-artifact@v4 + with: + name: benchmarking-results-clang + path: build-clang-18/benchmarks/benchmarking-results.json diff --git a/.github/workflows/bvt-gcc.yml b/.github/workflows/bvt-gcc.yml index 9979aed..e66981c 100644 --- a/.github/workflows/bvt-gcc.yml +++ b/.github/workflows/bvt-gcc.yml @@ -9,7 +9,7 @@ jobs: bvt-gcc: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} @@ -50,4 +50,11 @@ jobs: - name: run benchmarks run: | - ./build-gcc-14/benchmarks/msft_proxy_benchmarks --benchmark_repetitions=100 --benchmark_report_aggregates_only=true --benchmark_min_time=0.1s --benchmark_enable_random_interleaving=true + cd build-gcc-14/benchmarks + ./msft_proxy_benchmarks --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_enable_random_interleaving=true --benchmark_out=benchmarking-results.json + + - name: archive benchmarking results + uses: actions/upload-artifact@v4 + with: + name: benchmarking-results-gcc + path: build-gcc-14/benchmarks/benchmarking-results.json diff --git a/.github/workflows/bvt-msvc.yml b/.github/workflows/bvt-msvc.yml index e6c1982..d639a7f 100644 --- a/.github/workflows/bvt-msvc.yml +++ b/.github/workflows/bvt-msvc.yml @@ -9,7 +9,7 @@ jobs: bvt-msvc: runs-on: windows-2022 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} @@ -21,4 +21,11 @@ jobs: - name: run benchmarks run: | - .\build\benchmarks\Release\msft_proxy_benchmarks.exe --benchmark_repetitions=100 --benchmark_report_aggregates_only=true --benchmark_min_time=0.1s --benchmark_enable_random_interleaving=true + cd build\benchmarks + .\Release\msft_proxy_benchmarks.exe --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_enable_random_interleaving=true --benchmark_out=benchmarking-results.json + + - name: archive benchmarking results + uses: actions/upload-artifact@v4 + with: + name: benchmarking-results-msvc + path: build/benchmarks/benchmarking-results.json diff --git a/.github/workflows/bvt-report.yml b/.github/workflows/bvt-report.yml new file mode 100644 index 0000000..d910cf8 --- /dev/null +++ b/.github/workflows/bvt-report.yml @@ -0,0 +1,35 @@ +on: + workflow_call: + inputs: + branch: + type: string + required: false + +jobs: + bvt-report: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: download all workflow run artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: build report generator + run: | + cd tools/report_generator + cmake -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build -j + + - name: generate report + run: | + tools/report_generator/build/report_generator tools/report_generator/report-config.json ${{ github.sha }} artifacts benchmarking-report.md + + - name: archive benchmarking report + uses: actions/upload-artifact@v4 + with: + name: benchmarking-report + path: benchmarking-report.md diff --git a/.github/workflows/pipeline-ci.yml b/.github/workflows/pipeline-ci.yml index 17903bb..bae2c0a 100644 --- a/.github/workflows/pipeline-ci.yml +++ b/.github/workflows/pipeline-ci.yml @@ -6,10 +6,6 @@ on: pull_request: branches: [ main, release/** ] -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - jobs: run-bvt-gcc: uses: ./.github/workflows/bvt-gcc.yml @@ -26,3 +22,8 @@ jobs: run-bvt-appleclang: uses: ./.github/workflows/bvt-appleclang.yml name: Run BVT with AppleClang + + report: + uses: ./.github/workflows/bvt-report.yml + name: Generate report + needs: [run-bvt-gcc, run-bvt-clang, run-bvt-msvc, run-bvt-appleclang] diff --git a/.github/workflows/pipeline-release.yml b/.github/workflows/pipeline-release.yml index ba3ee90..0a7386c 100644 --- a/.github/workflows/pipeline-release.yml +++ b/.github/workflows/pipeline-release.yml @@ -65,6 +65,11 @@ jobs: with: branch: release/${{ github.event.inputs.version }} + report: + uses: ./.github/workflows/bvt-report.yml + name: Generate report + needs: [run-bvt-gcc, run-bvt-clang, run-bvt-msvc, run-bvt-appleclang] + draft-release: runs-on: windows-latest needs: [run-bvt-gcc, run-bvt-clang, run-bvt-msvc, run-bvt-appleclang] diff --git a/benchmarks/proxy_management_benchmark.cpp b/benchmarks/proxy_management_benchmark.cpp index 07d05fa..318bc52 100644 --- a/benchmarks/proxy_management_benchmark.cpp +++ b/benchmarks/proxy_management_benchmark.cpp @@ -87,7 +87,7 @@ void BM_SmallObjectManagementWithSharedPtr(benchmark::State& state) { } void BM_SmallObjectManagementWithSharedPtr_Pooled(benchmark::State& state) { - std::pmr::unsynchronized_pool_resource pool; + static std::pmr::unsynchronized_pool_resource pool; std::pmr::polymorphic_allocator<> alloc{&pool}; for (auto _ : state) { std::vector> data; @@ -128,7 +128,7 @@ void BM_LargeObjectManagementWithProxy(benchmark::State& state) { } void BM_LargeObjectManagementWithProxy_Pooled(benchmark::State& state) { - std::pmr::unsynchronized_pool_resource pool; + static std::pmr::unsynchronized_pool_resource pool; std::pmr::polymorphic_allocator<> alloc{&pool}; for (auto _ : state) { std::vector> data; @@ -169,7 +169,7 @@ void BM_LargeObjectManagementWithSharedPtr(benchmark::State& state) { } void BM_LargeObjectManagementWithSharedPtr_Pooled(benchmark::State& state) { - std::pmr::unsynchronized_pool_resource pool; + static std::pmr::unsynchronized_pool_resource pool; std::pmr::polymorphic_allocator<> alloc{&pool}; for (auto _ : state) { std::vector> data; diff --git a/tools/report_generator/CMakeLists.txt b/tools/report_generator/CMakeLists.txt new file mode 100644 index 0000000..bd99f38 --- /dev/null +++ b/tools/report_generator/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 3.5) + +project(report_generator) + +include(FetchContent) +# The policy uses the download time for timestamp, instead of the timestamp in the archive. This +# allows for proper rebuilds when a projects URL changes. +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() + +FetchContent_Declare( + nlohmann_json + URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz + URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d +) +FetchContent_MakeAvailable(nlohmann_json) + +add_executable(report_generator main.cpp) + +target_link_libraries(report_generator PRIVATE nlohmann_json::nlohmann_json) + +target_compile_features(report_generator PRIVATE cxx_std_20) +if (MSVC) + target_compile_options(report_generator PRIVATE /W4 /WX) +else() + target_compile_options(report_generator PRIVATE -Wall -Wextra -Wpedantic -Werror) +endif() diff --git a/tools/report_generator/main.cpp b/tools/report_generator/main.cpp new file mode 100644 index 0000000..2bde1bb --- /dev/null +++ b/tools/report_generator/main.cpp @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +struct EnvironmentInfo { + std::string Name; + std::string Description; + + friend void to_json(nlohmann::json& j, const EnvironmentInfo& e) { + j = nlohmann::json{ + {"Name", e.Name}, + {"Description", e.Description} + }; + } + + friend void from_json(const nlohmann::json& j, EnvironmentInfo& e) { + j.at("Name").get_to(e.Name); + j.at("Description").get_to(e.Description); + } +}; + +struct MetricInfo { + std::string Name; + std::string TargetBenchmarkName; + std::string BaselineBenchmarkName; + + friend void to_json(nlohmann::json& j, const MetricInfo& m) { + j = nlohmann::json{ + {"Name", m.Name}, + {"TargetBenchmarkName", m.TargetBenchmarkName}, + {"BaselineBenchmarkName", m.BaselineBenchmarkName} + }; + } + + friend void from_json(const nlohmann::json& j, MetricInfo& m) { + j.at("Name").get_to(m.Name); + j.at("TargetBenchmarkName").get_to(m.TargetBenchmarkName); + j.at("BaselineBenchmarkName").get_to(m.BaselineBenchmarkName); + } +}; + +struct ReportConfig { + std::string TargetName; + double YellowIndicatorThreshold; + std::vector Environments; + std::vector Metrics; + + friend void to_json(nlohmann::json& j, const ReportConfig& rc) { + j = nlohmann::json{ + {"TargetName", rc.TargetName}, + {"YellowIndicatorThreshold", rc.YellowIndicatorThreshold}, + {"Environments", rc.Environments}, + {"Metrics", rc.Metrics} + }; + } + + friend void from_json(const nlohmann::json& j, ReportConfig& rc) { + j.at("TargetName").get_to(rc.TargetName); + j.at("YellowIndicatorThreshold").get_to(rc.YellowIndicatorThreshold); + j.at("Environments").get_to(rc.Environments); + j.at("Metrics").get_to(rc.Metrics); + } +}; + +const std::string_view MedianSuffix = "_median"; + +std::unordered_map Parse(const std::filesystem::path& file) { + nlohmann::json obj; + { + std::ifstream in; + in.exceptions(std::ios_base::failbit | std::ios_base::badbit); + in.open(file, std::ios_base::in | std::ios_base::binary); + in >> obj; + } + std::unordered_map result; + for (auto& node : obj["benchmarks"]) { + std::string name = node["name"]; + if (name.ends_with(MedianSuffix)) { + name.resize(name.size() - MedianSuffix.size()); + double value = node["real_time"]; + result.emplace(std::move(name), value); + } + } + return result; +} + +void GenerateReport(const std::filesystem::path& config_path, const std::string& commit_id, const std::filesystem::path& source, const std::filesystem::path& output) { + ReportConfig config; + { + nlohmann::json obj; + std::ifstream in; + in.exceptions(std::ios_base::failbit | std::ios_base::badbit); + in.open(config_path, std::ios_base::in | std::ios_base::binary); + in >> obj; + obj.get_to(config); + } + std::vector> benchmarks; + benchmarks.reserve(config.Environments.size()); + for (auto& environment : config.Environments) { + benchmarks.push_back(Parse(source / std::format("benchmarking-results-{}", environment.Name) / "benchmarking-results.json")); + } + std::ofstream out; + out.exceptions(std::ios_base::failbit | std::ios_base::badbit); + out.open(output, std::ios_base::out | std::ios_base::trunc | std::ios_base::binary); + out << "## Benchmarking Report\n"; + out << "\n"; + out << "- Generated for: [Microsoft \"Proxy\" library](https://github.com/microsoft/proxy)\n"; + out << "- Commit ID: [" << commit_id << "](https://github.com/microsoft/proxy/commit/" << commit_id << ")\n"; + out << "- Generated at: " << std::format("{:%FT%TZ}", std::chrono::utc_clock::now()) << "\n"; + out << "\n"; + out << "| |"; + for (auto& environment : config.Environments) { + out << " " << environment.Description << " |"; + } + out << "\n"; + out << "| - |"; + for (std::size_t i = 0; i < config.Environments.size(); ++i) { + out << " - |"; + } + out << "\n"; + for (auto& metric : config.Metrics) { + out << "| " << metric.Name << " |"; + for (auto& benchmark : benchmarks) { + double target = benchmark.at(metric.TargetBenchmarkName); + double baseline = benchmark.at(metric.BaselineBenchmarkName); + double rate = (baseline - target) * 100 / target; + bool is_negative = rate < 0; + if (is_negative) { + rate = -rate; + } + out << " "; + if (rate < config.YellowIndicatorThreshold) { + out << "\xf0\x9f\x9f\xa1"; // Yellow circle + } else if (is_negative) { + out << "\xf0\x9f\x94\xb4"; // Red circle + } else { + out << "\xf0\x9f\x9f\xa2"; // Green circle + } + auto rate_str = std::format("{:.1f}", rate); + std::string message; + if (rate_str == "0.0") { + out << config.TargetName << " has similar performance"; + } else { + out << config.TargetName << " is about **" << rate_str << "% " << (is_negative ? "slower" : "faster") << "**"; + } + out << " |"; + } + out << "\n"; + } +} + +int main(int argc, char** argv) { + if (argc != 5) { + puts("Usage: report_generator "); + return 0; + } + try { + GenerateReport(argv[1], argv[2], argv[3], argv[4]); + } catch (const std::exception& e) { + fprintf(stderr, "An error occurred: %s\n", e.what()); + return 1; + } + return 0; +} diff --git a/tools/report_generator/report-config.json b/tools/report_generator/report-config.json new file mode 100644 index 0000000..47dc8d8 --- /dev/null +++ b/tools/report_generator/report-config.json @@ -0,0 +1,84 @@ +{ + "TargetName": "`proxy`", + "YellowIndicatorThreshold": 5.0, + "Environments": [ + { + "Name": "msvc", + "Description": "MSVC on Windows Server 2022 (x64)" + }, + { + "Name": "gcc", + "Description": "GCC on Ubuntu 24.04 (x64)" + }, + { + "Name": "clang", + "Description": "Clang on Ubuntu 24.04 (x64)" + }, + { + "Name": "appleclang", + "Description": "Apple Clang on macOS 15 (ARM64)" + } + ], + "Metrics": [ + { + "Name": "Indirect invocation on small objects via `proxy` vs. virtual functions", + "TargetBenchmarkName": "BM_SmallObjectInvocationViaProxy", + "BaselineBenchmarkName": "BM_SmallObjectInvocationViaVirtualFunction" + }, + { + "Name": "Indirect invocation on large objects via `proxy` vs. virtual functions", + "TargetBenchmarkName": "BM_LargeObjectInvocationViaProxy", + "BaselineBenchmarkName": "BM_LargeObjectInvocationViaVirtualFunction" + }, + { + "Name": "Basic lifetime management for small objects with `proxy` vs. `std::unique_ptr`", + "TargetBenchmarkName": "BM_SmallObjectManagementWithProxy", + "BaselineBenchmarkName": "BM_SmallObjectManagementWithUniquePtr" + }, + { + "Name": "Basic lifetime management for small objects with `proxy` vs. `std::shared_ptr` (without memory pool)", + "TargetBenchmarkName": "BM_SmallObjectManagementWithProxy", + "BaselineBenchmarkName": "BM_SmallObjectManagementWithSharedPtr" + }, + { + "Name": "Basic lifetime management for small objects with `proxy` vs. `std::shared_ptr` (with memory pool)", + "TargetBenchmarkName": "BM_SmallObjectManagementWithProxy", + "BaselineBenchmarkName": "BM_SmallObjectManagementWithSharedPtr_Pooled" + }, + { + "Name": "Basic lifetime management for small objects with `proxy` vs. `std::any`", + "TargetBenchmarkName": "BM_SmallObjectManagementWithProxy", + "BaselineBenchmarkName": "BM_SmallObjectManagementWithAny" + }, + { + "Name": "Basic lifetime management for large objects with `proxy` (without memory pool) vs. `std::unique_ptr`", + "TargetBenchmarkName": "BM_LargeObjectManagementWithProxy", + "BaselineBenchmarkName": "BM_LargeObjectManagementWithUniquePtr" + }, + { + "Name": "Basic lifetime management for large objects with `proxy` (with memory pool) vs. `std::unique_ptr`", + "TargetBenchmarkName": "BM_LargeObjectManagementWithProxy_Pooled", + "BaselineBenchmarkName": "BM_LargeObjectManagementWithUniquePtr" + }, + { + "Name": "Basic lifetime management for large objects with `proxy` vs. `std::shared_ptr` (both without memory pool)", + "TargetBenchmarkName": "BM_LargeObjectManagementWithProxy", + "BaselineBenchmarkName": "BM_LargeObjectManagementWithSharedPtr" + }, + { + "Name": "Basic lifetime management for large objects with `proxy` vs. `std::shared_ptr` (both with memory pool)", + "TargetBenchmarkName": "BM_LargeObjectManagementWithProxy_Pooled", + "BaselineBenchmarkName": "BM_LargeObjectManagementWithSharedPtr_Pooled" + }, + { + "Name": "Basic lifetime management for large objects with `proxy` (without memory pool) vs. `std::any`", + "TargetBenchmarkName": "BM_LargeObjectManagementWithProxy", + "BaselineBenchmarkName": "BM_LargeObjectManagementWithAny" + }, + { + "Name": "Basic lifetime management for large objects with `proxy` (with memory pool) vs. `std::any`", + "TargetBenchmarkName": "BM_LargeObjectManagementWithProxy_Pooled", + "BaselineBenchmarkName": "BM_LargeObjectManagementWithAny" + } + ] +} \ No newline at end of file