[benchmark] add gbenchmark

.gitignore

@@ -10,3 +10,4 @@ __pycache__/
 /out/
 /tools/out/
 /CMakeLists.txt.user
+/*.log

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "boost-math"]
 	path = boost-math
 	url = https://github.com/boostorg/math.git
+[submodule "benchmarks/google-benchmark"]
+	path = benchmarks/google-benchmark
+	url = https://github.com/google/benchmark.git

.vscode/settings.json

@@ -2,14 +2,18 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 {
     "C_Cpp.autoAddFileAssociations": false,
-    "editor.formatOnSave": true,
+    "editor.formatOnSave": false,
+    "[cpp]": {
+        "editor.formatOnSave": true
+    },
     "files.associations": {
         ".clang-format": "yaml",
         "header-units.json": "jsonc",
         "**/stl/inc/**": "cpp"
     },
     "files.eol": "\r\n",
     "files.exclude": {
+        "benchmarks/google-benchmark": true,
         "llvm-project": true,
         "stl/msbuild": true,
         "boost-math": true

CMakeLists.txt

@@ -38,15 +38,9 @@ if(TARGET run-validate)
 endif()
 
 option(BUILD_TESTING "Enable testing" ON)
+option(STL_BUILD_BENCHMARKING "Enable benchmarking" OFF)
 set(VCLIBS_SUFFIX "_oss" CACHE STRING "suffix for built DLL names to avoid conflicts with distributed DLLs")
 
-set(CMAKE_CXX_FLAGS "")
-set(CMAKE_CXX_FLAGS_DEBUG "")
-set(CMAKE_CXX_FLAGS_RELEASE "")
-set(CMAKE_CXX_STANDARD_LIBRARIES "kernel32.lib")
-set(CMAKE_CXX_STANDARD_LIBRARIES_INIT "kernel32.lib")
-set(CMAKE_MSVC_RUNTIME_LIBRARY "")
-
 if("${VCLIBS_TARGET_ARCHITECTURE}" MATCHES "^x86$")
     set(VCLIBS_TARGET_ARCHITECTURE "x86")
     set(VCLIBS_I386_OR_AMD64 "i386")
@@ -65,26 +59,27 @@ elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^armv7$")
     set(VCLIBS_I386_OR_AMD64 "arm")
     set(VCLIBS_X86_OR_X64 "arm")
     add_compile_definitions(_ARM_ _VCRT_WIN32_WINNT=0x0602 _STL_WIN32_WINNT=0x0602)
-    string(APPEND CMAKE_CXX_STANDARD_LIBRARIES " Synchronization.lib")
 elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^arm64$")
     set(VCLIBS_TARGET_ARCHITECTURE "arm64")
     set(VCLIBS_I386_OR_AMD64 "arm64")
     set(VCLIBS_X86_OR_X64 "arm64")
     add_compile_definitions(_ARM64_ _VCRT_WIN32_WINNT=0x0A00 _STL_WIN32_WINNT=0x0A00)
-    string(APPEND CMAKE_CXX_STANDARD_LIBRARIES " Synchronization.lib")
 else()
     message(FATAL_ERROR "Could not determine target architecture: VCLIBS_TARGET_ARCHITECTURE: ${VCLIBS_TARGET_ARCHITECTURE}")
 endif()
 
+get_filename_component(TOOLSET_BINARIES_DIR "${CMAKE_CXX_COMPILER}" DIRECTORY) # Example: $\VC\Tools\MSVC\14.23.27931\bin\Hostx86\x86
+get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_BINARIES_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931\bin\Hostx86
+get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931\bin
+get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931
+set(TOOLSET_LIB "${TOOLSET_ROOT_DIR}/lib/${VCLIBS_X86_OR_X64}")
+set(STL_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
+set(STL_LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
+set(STL_RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/bin/${VCLIBS_I386_OR_AMD64}")
 add_compile_definitions(
     _ALLOW_ITERATOR_DEBUG_LEVEL_MISMATCH WIN32_LEAN_AND_MEAN STRICT _CRT_STDIO_ARBITRARY_WIDE_SPECIFIERS
     _CRT_DECLARE_NONSTDC_NAMES=1 _WIN32_WINNT=0x0A00 NTDDI_VERSION=NTDDI_WIN10_CO)
 
-add_compile_options(/WX /Gy
-    "$<$<COMPILE_LANGUAGE:CXX>:/diagnostics:caret;/W4;/w14265;/w15038;/fastfail;/guard:cf;/Z7;/Zp8;/std:c++latest;/permissive-;/Zc:threadSafeInit-;/Zl>"
-    # note that /Zi generates debug info inside the object file, it's the same as /Z7 for msvc
-    "$<$<COMPILE_LANGUAGE:ASM_MASM>:/Zi;/W3;/nologo>")
-
 set(VCLIBS_DEBUG_OPTIONS "$<$<COMPILE_LANGUAGE:CXX>:/Od>")
 
 # TRANSITION: Potentially remove `/Os` if it results in speed improvements.
@@ -96,28 +91,16 @@ set(VCLIBS_DEBUG_OPTIONS "$<$<COMPILE_LANGUAGE:CXX>:/Od>")
 # See GH-2108 for more info.
 set(VCLIBS_RELEASE_OPTIONS "$<$<COMPILE_LANGUAGE:CXX>:/O2;/Os>")
 
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/bin/${VCLIBS_I386_OR_AMD64}")
-
-set(CMAKE_STATIC_LINKER_FLAGS "/WX")
-set(CMAKE_STATIC_LINKER_FLAGS_DEBUG "")
-set(CMAKE_STATIC_LINKER_FLAGS_RELEASE "")
-set(CMAKE_SHARED_LINKER_FLAGS "/DEBUG:FULL /WX /RELEASE /SUBSYSTEM:Console /NODEFAULTLIB /INCREMENTAL:NO /MANIFEST:NO /DLL /profile /guard:cf /DEBUGTYPE:cv,fixup /LARGEADDRESSAWARE")
-set(CMAKE_SHARED_LINKER_FLAGS_DEBUG "")
-set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "")
-
-get_filename_component(TOOLSET_BINARIES_DIR "${CMAKE_CXX_COMPILER}" DIRECTORY) # Example: $\VC\Tools\MSVC\14.23.27931\bin\Hostx86\x86
-get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_BINARIES_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931\bin\Hostx86
-get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931\bin
-get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931
-
-set(TOOLSET_LIB "${TOOLSET_ROOT_DIR}/lib/${VCLIBS_X86_OR_X64}")
-
 add_subdirectory(boost-math)
 add_subdirectory(stl)
 
 if(BUILD_TESTING)
     enable_testing()
     add_subdirectory(tests)
 endif()
+if(STL_BUILD_BENCHMARKING)
+    if(VCLIBS_TARGET_ARCHITECTURE STREQUAL "arm")
+        message(WARNING "google benchmark does not build on arm32 - this build will likely fail.")
+    endif()
+    add_subdirectory(benchmarks)
+endif()

README.md

@@ -397,6 +397,42 @@ build folder to your path:
 set PATH=C:\STL\out\build\x64\out\bin\amd64;%PATH%
 ```
 
+# Benchmarking
+
+For performance sensitive code – containers, algorithms, and the like –
+you may wish to write and/or run benchmarks, and the STL team will likely
+run any benchmarks we do have in our PR process. Additionally,
+if you are writing a "performance improvement" PR, please add and run benchmarks
+to show that the PR does, in fact, improve performance.
+
+The benchmarking code is located in `benchmarks`. Adding a new benchmark is as easy as adding a new file
+to `benchmarks/src`, and then adding `add_benchmark(<name> <source_file>)`
+to `benchmarks/CMakeLists.txt`.
+You may also modify an existing benchmark file. We use Google's [Benchmark][gbenchmark] library,
+so you may find [their documentation][gbenchmark:docs] helpful, and you can also read the existing code
+for how _we_ use it.
+
+To run benchmarks, you'll need to configure the STL with the `-DSTL_BUILD_BENCHMARKING=ON` option:
+
+```cmd
+> cmake -B out\bench -S . -G Ninja -DSTL_BUILD_BENCHMARKING=ON
+> cmake --build out\bench
+```
+
+you can then run your benchmark with:
+
+```cmd
+> out\bench\benchmarks\benchmark-<benchmark-name> --benchmark_out=<file> --benchmark_out_format=csv
+```
+
+And then you can copy this csv file into Excel, or another spreadsheet program.
+
+If you want to see all the other flags you can pass, run:
+
+```cmd
+> out\bench\benchmarks\benchmark-<benchmark-name> --help
+```
+
 # Editing And Testing The Debugger Visualizer
 
 ### Modify The Visualizer

azure-devops/checkout-sources.yml

@@ -8,6 +8,9 @@ parameters:
 - name: boostMathSHAVar
   type: string
   default: boostMathSHA
+- name: googleBenchmarkSHAVar
+  type: string
+  default: googleBenchmarkSHA
 steps:
 - checkout: self
   clean: true
@@ -24,6 +27,9 @@ steps:
       Write-Host "##vso[task.setvariable variable=${{ parameters.llvmSHAVar }};]$llvmSHA"
       $boostMathSHA = git submodule status --cached boost-math | %{$_ -replace $regexSubmoduleSHA, '$1'}
       Write-Host "##vso[task.setvariable variable=${{ parameters.boostMathSHAVar }};]$boostMathSHA"
+      $googleBenchmarkSHA = git submodule status --cached benchmarks/google-benchmark `
+        | %{$_ -replace $regexSubmoduleSHA, '$1'}
+      Write-Host "##vso[task.setvariable variable=${{ parameters.googleBenchmarkSHAVar }};]$googleBenchmarkSHA"
 - script: |
     cd $(Build.SourcesDirectory)
     if not exist "llvm-project" (
@@ -66,3 +72,23 @@ steps:
     git fetch --filter=tree:0 --depth=1 boostorg $(${{ parameters.boostMathSHAVar }})
     git checkout -f FETCH_HEAD
   displayName: "Checkout boost-math source"
+- script: |
+    cd $(Build.SourcesDirectory)/benchmarks
+    if not exist "google-benchmark" (
+      mkdir google-benchmark
+    )
+    cd google-benchmark
+
+    if not exist ".git" (
+      del /S /Q *
+      git init
+    )
+
+    git remote get-url googlebenchmark
+    if errorlevel 1 (
+      git remote add googlebenchmark https://github.com/google/benchmark.git
+    )
+
+    git fetch --filter=tree:0 --depth=1 googlebenchmark $(${{ parameters.googleBenchmarkSHAVar }})
+    git checkout -f FETCH_HEAD
+  displayName: "Checkout google benchmark source"

azure-devops/cmake-configure-build.yml

@@ -11,6 +11,9 @@ parameters:
 - name: buildOutputLocationVar
   type: string
   default: buildOutputLocation
+- name: buildBenchmarking
+  type: string
+  default: 'ON'
 - name: cmakeAdditionalFlags
   type: string
   default: ''
@@ -32,8 +35,8 @@ steps:
     cmake ${{ parameters.cmakeAdditionalFlags}} -G Ninja ^
     -DCMAKE_CXX_COMPILER=cl ^
     -DCMAKE_BUILD_TYPE=Release ^
+    -DSTL_BUILD_BENCHMARKING=${{ parameters.buildBenchmarking }} ^
     -DLIT_FLAGS=$(litFlags) ^
-    -DCMAKE_CXX_FLAGS=/analyze:autolog- ^
     -S $(Build.SourcesDirectory) -B $(${{ parameters.buildOutputLocationVar }})
   displayName: 'Configure the STL'
   timeoutInMinutes: 2

azure-devops/cross-build.yml

@@ -15,6 +15,9 @@ parameters:
 - name: numShards
   type: number
   default: 8
+- name: buildBenchmarking
+  type: string
+  default: 'ON'
 jobs:
 - job: '${{ parameters.targetPlatform }}'
   variables:
@@ -38,6 +41,7 @@ jobs:
       targetPlatform: ${{ parameters.targetPlatform }}
       hostArch: ${{ parameters.hostArch }}
       targetArch: ${{ parameters.vsDevCmdArch }}
+      buildBenchmarking: ${{ parameters.buildBenchmarking }}
       cmakeAdditionalFlags: '-DTESTS_BUILD_ONLY=ON'
   - template: run-tests.yml
     parameters:

azure-pipelines.yml

@@ -82,6 +82,7 @@ stages:
         parameters:
           targetPlatform: arm
           vsDevCmdArch: arm
+          buildBenchmarking: 'OFF'
 
   - stage: Build_ARM64
     dependsOn: Build_And_Test_x64

benchmarks/CMakeLists.txt

@@ -0,0 +1,75 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set(STL_BENCHMARK_MSVC_RUNTIME_LIBRARY
+    MultiThreaded
+    CACHE STRING "The flavor of the standard library to use; see https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html for more information.")
+set_property(CACHE STL_BENCHMARK_MSVC_RUNTIME_LIBRARY
+    PROPERTY STRINGS
+    "MultiThreaded;MultiThreadedDLL;MultiThreadedDebug;MultiThreadedDebugDLL"
+)
+set(CMAKE_MSVC_RUNTIME_LIBRARY "${STL_BENCHMARK_MSVC_RUNTIME_LIBRARY}")
+
+set(STL_BENCHMARK_ITERATOR_DEBUG_LEVEL
+    default
+    CACHE STRING "What level of iterator debugging to use."
+)
+set_property(CACHE STL_BENCHMARK_ITERATOR_DEBUG_LEVEL
+    PROPERTY STRINGS
+    "default;0;1;2"
+)
+
+if(NOT STL_BENCHMARK_ITERATOR_DEBUG_LEVEL STREQUAL "default")
+    add_compile_definitions("_ITERATOR_DEBUG_LEVEL=${STL_BENCHMARK_ITERATOR_DEBUG_LEVEL}")
+endif()
+
+set(CMAKE_BUILD_TYPE RelWithDebInfo)
+
+if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/google-benchmark/.git")
+    message(FATAL_ERROR "google-benchmark is not checked out; make sure to run\n    git submodule update --init benchmarks/google-benchmark")
+endif()
+
+set(BENCHMARK_ENABLE_DOXYGEN OFF)
+set(BENCHMARK_ENABLE_INSTALL OFF)
+set(BENCHMARK_ENABLE_TESTING OFF)
+set(BUILD_SHARED_LIBS OFF)
+
+# TRANSITION, GH-2816: on some machines, librt is found, despite it being a unix-only library
+set(HAVE_LIB_RT OFF)
+
+include_directories(BEFORE "${CMAKE_BINARY_DIR}/out/inc")
+link_directories(BEFORE "${STL_LIBRARY_OUTPUT_DIRECTORY}")
+
+add_subdirectory(google-benchmark EXCLUDE_FROM_ALL)
+
+set(benchmark_headers
+    "inc/udt.hpp"
+)
+
+function(add_benchmark name)
+    cmake_parse_arguments(PARSE_ARGV 1 "arg" "" "CXX_STANDARD" "")
+
+    if(NOT DEFINED arg_CXX_STANDARD)
+        set(arg_CXX_STANDARD 23)
+    elseif(NOT arg_CXX_STANDARD MATCHES "^[0-9][0-9]$")
+        message(FATAL_ERROR "Unexpected value for CXX_STANDARD: ${arg_CXX_STANDARD}")
+    endif()
+
+    if(NOT DEFINED arg_UNPARSED_ARGUMENTS)
+        message(FATAL_ERROR "benchmark ${name} does not have any source files")
+    endif()
+
+    add_executable(benchmark-${name}
+        ${benchmark_headers}
+        ${arg_UNPARSED_ARGUMENTS}
+    )
+
+    target_compile_features(benchmark-${name} PRIVATE cxx_std_${arg_CXX_STANDARD})
+    target_include_directories(benchmark-${name} PRIVATE inc)
+    target_link_libraries(benchmark-${name} PRIVATE benchmark::benchmark)
+endfunction()
+
+add_benchmark(std_copy
+    src/std_copy.cpp
+    CXX_STANDARD 23
+)

benchmarks/inc/udt.hpp

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+template <typename Contained>
+struct aggregate {
+    Contained c;
+
+    friend bool operator==(const aggregate&, const aggregate&) = default;
+};
+
+template <typename Contained>
+struct non_trivial {
+    Contained c;
+    non_trivial() : c() {}
+    non_trivial(const Contained& src) : c(src) {}
+    non_trivial(const non_trivial& other) : c(other.c) {}
+    non_trivial& operator=(const non_trivial& other) {
+        c = other.c;
+        return *this;
+    }
+    ~non_trivial() {}
+
+    friend bool operator==(const non_trivial&, const non_trivial&) = default;
+};

benchmarks/inc/utility.hpp

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+#include <algorithm>
+#include <random>
+#include <vector>
+
+#include <xoshiro.hpp>
+
+template <class Contained>
+std::vector<Contained> random_vector(size_t n) {
+    std::random_device rd;
+    std::uniform_int_distribution<uint64_t> id64;
+    xoshiro256ss prng{id64(rd), id64(rd), id64(rd), id64(rd)};
+
+    std::vector<Contained> res(n);
+    std::generate(res.begin(), res.end(), [&prng] { return static_cast<Contained>(prng.next()); });
+    return res;
+}