[benchmark] add gbenchmark (#2780)

Co-authored-by: Nicole Mazzuca <[email protected]>
Co-authored-by: Stephan T. Lavavej <[email protected]>
Co-authored-by: Casey Carter <[email protected]>

.gitignore

@@ -10,3 +10,4 @@ __pycache__/
 /out/
 /tools/out/
 /CMakeLists.txt.user
+/*.log

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "boost-math"]
 	path = boost-math
 	url = https://github.com/boostorg/math.git
+[submodule "benchmarks/google-benchmark"]
+	path = benchmarks/google-benchmark
+	url = https://github.com/google/benchmark.git

.vscode/settings.json

@@ -2,14 +2,18 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 {
     "C_Cpp.autoAddFileAssociations": false,
-    "editor.formatOnSave": true,
+    "editor.formatOnSave": false,
+    "[cpp]": {
+        "editor.formatOnSave": true
+    },
     "files.associations": {
         ".clang-format": "yaml",
         "header-units.json": "jsonc",
         "**/stl/inc/**": "cpp"
     },
     "files.eol": "\r\n",
     "files.exclude": {
+        "benchmarks/google-benchmark": true,
         "llvm-project": true,
         "stl/msbuild": true,
         "boost-math": true

CMakeLists.txt

@@ -42,17 +42,11 @@ if(TARGET run-validate)
 endif()
 
 option(BUILD_TESTING "Enable testing" ON)
+option(STL_BUILD_BENCHMARKING "Enable benchmarking" OFF)
 set(VCLIBS_SUFFIX "_oss" CACHE STRING "suffix for built DLL names to avoid conflicts with distributed DLLs")
 
 option(STL_USE_ANALYZE "Pass the /analyze flag to MSVC" OFF)
 
-set(CMAKE_CXX_FLAGS "")
-set(CMAKE_CXX_FLAGS_DEBUG "")
-set(CMAKE_CXX_FLAGS_RELEASE "")
-set(CMAKE_CXX_STANDARD_LIBRARIES "kernel32.lib")
-set(CMAKE_CXX_STANDARD_LIBRARIES_INIT "kernel32.lib")
-set(CMAKE_MSVC_RUNTIME_LIBRARY "")
-
 if("${VCLIBS_TARGET_ARCHITECTURE}" MATCHES "^x86$")
     set(VCLIBS_TARGET_ARCHITECTURE "x86")
     set(VCLIBS_I386_OR_AMD64 "i386")
@@ -72,27 +66,27 @@ elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^armv7$")
     set(VCLIBS_I386_OR_AMD64 "arm")
     set(VCLIBS_X86_OR_X64 "arm")
     add_compile_definitions(_ARM_ _VCRT_WIN32_WINNT=0x0602 _STL_WIN32_WINNT=0x0602)
-    string(APPEND CMAKE_CXX_STANDARD_LIBRARIES " Synchronization.lib")
 elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^arm64$")
     set(VCLIBS_TARGET_ARCHITECTURE "arm64")
     set(VCLIBS_I386_OR_AMD64 "arm64")
     set(VCLIBS_X86_OR_X64 "arm64")
     add_compile_definitions(_ARM64_ _VCRT_WIN32_WINNT=0x0A00 _STL_WIN32_WINNT=0x0A00)
-    string(APPEND CMAKE_CXX_STANDARD_LIBRARIES " Synchronization.lib")
 else()
     message(FATAL_ERROR "Could not determine target architecture: VCLIBS_TARGET_ARCHITECTURE: ${VCLIBS_TARGET_ARCHITECTURE}")
 endif()
 
+get_filename_component(TOOLSET_BINARIES_DIR "${CMAKE_CXX_COMPILER}" DIRECTORY) # Example: $\VC\Tools\MSVC\14.23.27931\bin\Hostx86\x86
+get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_BINARIES_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931\bin\Hostx86
+get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931\bin
+get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931
+set(TOOLSET_LIB "${TOOLSET_ROOT_DIR}/lib/${VCLIBS_X86_OR_X64}")
+set(STL_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
+set(STL_LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
+set(STL_RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/bin/${VCLIBS_I386_OR_AMD64}")
 add_compile_definitions(
     _ALLOW_ITERATOR_DEBUG_LEVEL_MISMATCH WIN32_LEAN_AND_MEAN STRICT _CRT_STDIO_ARBITRARY_WIDE_SPECIFIERS
     _CRT_DECLARE_NONSTDC_NAMES=1 _WIN32_WINNT=0x0A00 NTDDI_VERSION=NTDDI_WIN10_CO)
 
-add_compile_options(/WX /Gy
-    "$<$<COMPILE_LANGUAGE:CXX>:/diagnostics:caret;/W4;/w14265;/w15038;/fastfail;/guard:cf;/Z7;/Zp8;/std:c++latest;/permissive-;/Zc:threadSafeInit-;/Zl>"
-
-    # note that /Zi generates debug info inside the object file, it's the same as /Z7 for msvc
-    "$<$<COMPILE_LANGUAGE:ASM_MASM>:/Zi;/W3;/nologo>")
-
 if(STL_USE_ANALYZE)
     # TRANSITION OS-40109504: Windows SDK: incorrect SAL annotations on functions the STL uses
     # warning C6553: The annotation for function 'LCMapStringEx' on _Param_(9)
@@ -114,28 +108,16 @@ set(VCLIBS_DEBUG_OPTIONS "$<$<COMPILE_LANGUAGE:CXX>:/Od>")
 # See GH-2108 for more info.
 set(VCLIBS_RELEASE_OPTIONS "$<$<COMPILE_LANGUAGE:CXX>:/O2;/Os>")
 
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/lib/${VCLIBS_I386_OR_AMD64}")
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/out/bin/${VCLIBS_I386_OR_AMD64}")
-
-set(CMAKE_STATIC_LINKER_FLAGS "/WX")
-set(CMAKE_STATIC_LINKER_FLAGS_DEBUG "")
-set(CMAKE_STATIC_LINKER_FLAGS_RELEASE "")
-set(CMAKE_SHARED_LINKER_FLAGS "/DEBUG:FULL /WX /RELEASE /SUBSYSTEM:Console /NODEFAULTLIB /INCREMENTAL:NO /MANIFEST:NO /DLL /profile /guard:cf /DEBUGTYPE:cv,fixup /LARGEADDRESSAWARE")
-set(CMAKE_SHARED_LINKER_FLAGS_DEBUG "")
-set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "")
-
-get_filename_component(TOOLSET_BINARIES_DIR "${CMAKE_CXX_COMPILER}" DIRECTORY) # Example: $\VC\Tools\MSVC\14.23.27931\bin\Hostx86\x86
-get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_BINARIES_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931\bin\Hostx86
-get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931\bin
-get_filename_component(TOOLSET_ROOT_DIR "${TOOLSET_ROOT_DIR}" DIRECTORY) # $\VC\Tools\MSVC\14.23.27931
-
-set(TOOLSET_LIB "${TOOLSET_ROOT_DIR}/lib/${VCLIBS_X86_OR_X64}")
-
 add_subdirectory(boost-math)
 add_subdirectory(stl)
 
 if(BUILD_TESTING)
     enable_testing()
     add_subdirectory(tests)
 endif()
+if(STL_BUILD_BENCHMARKING)
+    if(VCLIBS_TARGET_ARCHITECTURE STREQUAL "arm")
+        message(WARNING "google benchmark does not build on arm32 - this build will likely fail.")
+    endif()
+    add_subdirectory(benchmarks)
+endif()

NOTICE.txt

@@ -216,3 +216,13 @@ In addition, certain files include the notices provided below.
 // shall not be used in advertising or otherwise to promote the sale,
 // use or other dealings in these Data Files or Software without prior
 // written authorization of the copyright holder.
+
+----------------------
+
+/*  Written in 2018 by David Blackman and Sebastiano Vigna ([email protected])
+
+To the extent possible under law, the author has dedicated all copyright
+and related and neighboring rights to this software to the public domain
+worldwide. This software is distributed without any warranty.
+
+See <http://creativecommons.org/publicdomain/zero/1.0/>. */

README.md

@@ -151,7 +151,7 @@ Just try to follow these rules, so we can spend more time fixing bugs and implem
 2. Open Visual Studio, and choose the "Clone or check out code" option. Enter the URL of this repository,
    `https://github.com/microsoft/STL`.
 3. Open a terminal in the IDE with `` Ctrl + ` `` (by default) or press on "View" in the top bar, and then "Terminal".
-4. In the terminal, invoke `git submodule update --init --progress llvm-project boost-math`
+4. In the terminal, invoke `git submodule update --init --progress`
 5. Choose the architecture you wish to build in the IDE, and build as you would any other project. All necessary CMake
    settings are set by `CMakeSettings.json`.
 
@@ -398,6 +398,46 @@ build folder to your path:
 set PATH=C:\STL\out\build\x64\out\bin\amd64;%PATH%
 ```
 
+# Benchmarking
+
+For performance-sensitive code – containers, algorithms, and the like –
+you may wish to write and/or run benchmarks, and the STL team will likely
+run any benchmarks we do have in our PR process. Additionally,
+if you are writing a "performance improvement" PR, please add and run benchmarks
+to show that the PR does, in fact, improve performance.
+
+The benchmarking code is located in `benchmarks`. Adding a new benchmark is as easy as adding a new file
+to `benchmarks/src`, and then adding `add_benchmark(<name> <source_file>)`
+to `benchmarks/CMakeLists.txt`.
+You may also modify an existing benchmark file. We use Google's [Benchmark][gbenchmark] library,
+so you may find [their documentation][gbenchmark:docs] helpful, and you can also read the existing code
+for how _we_ use it.
+
+To run benchmarks, you'll need to configure the STL with the `-DSTL_BUILD_BENCHMARKING=ON` option:
+
+```cmd
+cmake -B out\bench -S . -G Ninja -DSTL_BUILD_BENCHMARKING=ON
+cmake --build out\bench
+```
+
+You can then run your benchmark with:
+
+```cmd
+out\bench\benchmarks\benchmark-<benchmark-name> --benchmark_out=<file> --benchmark_out_format=csv
+```
+
+And then you can copy this csv file into Excel, or another spreadsheet program. For example:
+
+```cmd
+out\bench\benchmarks\benchmark-std_copy --benchmark_out=benchmark-std_copy-results.csv --benchmark_out_format=csv
+```
+
+If you want to see all the other flags you can pass, run:
+
+```cmd
+out\bench\benchmarks\benchmark-<benchmark-name> --help
+```
+
 # Editing And Testing The Debugger Visualizer
 
 ### Modify The Visualizer
@@ -500,6 +540,8 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 [bug tag]: https://github.com/microsoft/STL/issues?q=is%3Aopen+is%3Aissue+label%3Abug
 [cxx20 tag]: https://github.com/microsoft/STL/issues?q=is%3Aopen+is%3Aissue+label%3Acxx20
 [enhancement tag]: https://github.com/microsoft/STL/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement
+[gbenchmark]: https://github.com/google/benchmark
+[gbenchmark:docs]: https://github.com/google/benchmark/blob/main/docs/user_guide.md
 [hub]: https://support.microsoft.com/en-us/help/4021566/windows-10-send-feedback-to-microsoft-with-feedback-hub-app
 [libcxx]: https://libcxx.llvm.org
 [lit]: https://llvm.org/docs/CommandGuide/lit.html

azure-devops/checkout-sources.yml

@@ -8,6 +8,9 @@ parameters:
 - name: boostMathSHAVar
   type: string
   default: boostMathSHA
+- name: googleBenchmarkSHAVar
+  type: string
+  default: googleBenchmarkSHA
 steps:
 - checkout: self
   clean: true
@@ -28,6 +31,9 @@ steps:
       Write-Host "##vso[task.setvariable variable=${{ parameters.llvmSHAVar }};]$llvmSHA"
       $boostMathSHA = git submodule status --cached boost-math | %{$_ -replace $regexSubmoduleSHA, '$1'}
       Write-Host "##vso[task.setvariable variable=${{ parameters.boostMathSHAVar }};]$boostMathSHA"
+      $googleBenchmarkSHA = git submodule status --cached benchmarks/google-benchmark `
+        | %{$_ -replace $regexSubmoduleSHA, '$1'}
+      Write-Host "##vso[task.setvariable variable=${{ parameters.googleBenchmarkSHAVar }};]$googleBenchmarkSHA"
 - script: |
     cd $(Build.SourcesDirectory)
     if not exist "llvm-project" (
@@ -71,3 +77,24 @@ steps:
     git reset --quiet --hard FETCH_HEAD
     git clean --quiet -x -d -f -f
   displayName: "Checkout boost-math source"
+- script: |
+    cd $(Build.SourcesDirectory)/benchmarks
+    if not exist "google-benchmark" (
+      mkdir google-benchmark
+    )
+    cd google-benchmark
+
+    if not exist ".git" (
+      del /S /Q *
+      git init
+    )
+
+    git remote get-url googlebenchmark
+    if errorlevel 1 (
+      git remote add googlebenchmark https://github.com/google/benchmark.git
+    )
+
+    git fetch --filter=tree:0 --depth=1 googlebenchmark $(${{ parameters.googleBenchmarkSHAVar }})
+    git reset --quiet --hard FETCH_HEAD
+    git clean --quiet -x -d -f -f
+  displayName: "Checkout google benchmark source"

azure-devops/cmake-configure-build.yml

@@ -11,6 +11,9 @@ parameters:
 - name: buildOutputLocationVar
   type: string
   default: buildOutputLocation
+- name: buildBenchmarking
+  type: string
+  default: 'ON'
 - name: cmakeAdditionalFlags
   type: string
   default: ''
@@ -32,6 +35,7 @@ steps:
     cmake ${{ parameters.cmakeAdditionalFlags}} -G Ninja ^
     -DCMAKE_CXX_COMPILER=cl ^
     -DCMAKE_BUILD_TYPE=Release ^
+    -DSTL_BUILD_BENCHMARKING=${{ parameters.buildBenchmarking }} ^
     -DLIT_FLAGS=$(litFlags) ^
     -DSTL_USE_ANALYZE=ON ^
     -S $(Build.SourcesDirectory) -B $(${{ parameters.buildOutputLocationVar }})

azure-devops/cross-build.yml

@@ -15,6 +15,9 @@ parameters:
 - name: numShards
   type: number
   default: 8
+- name: buildBenchmarking
+  type: string
+  default: 'ON'
 jobs:
 - job: '${{ parameters.targetPlatform }}'
   variables:
@@ -38,6 +41,7 @@ jobs:
       targetPlatform: ${{ parameters.targetPlatform }}
       hostArch: ${{ parameters.hostArch }}
       targetArch: ${{ parameters.vsDevCmdArch }}
+      buildBenchmarking: ${{ parameters.buildBenchmarking }}
       cmakeAdditionalFlags: '-DTESTS_BUILD_ONLY=ON'
   - template: run-tests.yml
     parameters:

azure-pipelines.yml

@@ -86,6 +86,7 @@ stages:
         parameters:
           targetPlatform: arm
           vsDevCmdArch: arm
+          buildBenchmarking: 'OFF'
 
   - stage: Build_ARM64
     dependsOn: Build_And_Test_x64

benchmarks/CMakeLists.txt

@@ -0,0 +1,77 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set(STL_BENCHMARK_MSVC_RUNTIME_LIBRARY
+    MultiThreaded
+    CACHE STRING "The flavor of the standard library to use; see https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html for more information.")
+set_property(CACHE STL_BENCHMARK_MSVC_RUNTIME_LIBRARY
+    PROPERTY STRINGS
+    "MultiThreaded;MultiThreadedDLL;MultiThreadedDebug;MultiThreadedDebugDLL"
+)
+set(CMAKE_MSVC_RUNTIME_LIBRARY "${STL_BENCHMARK_MSVC_RUNTIME_LIBRARY}")
+
+set(STL_BENCHMARK_ITERATOR_DEBUG_LEVEL
+    default
+    CACHE STRING "What level of iterator debugging to use."
+)
+set_property(CACHE STL_BENCHMARK_ITERATOR_DEBUG_LEVEL
+    PROPERTY STRINGS
+    "default;0;1;2"
+)
+
+if(NOT STL_BENCHMARK_ITERATOR_DEBUG_LEVEL STREQUAL "default")
+    add_compile_definitions("_ITERATOR_DEBUG_LEVEL=${STL_BENCHMARK_ITERATOR_DEBUG_LEVEL}")
+endif()
+
+set(CMAKE_BUILD_TYPE RelWithDebInfo)
+
+if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/google-benchmark/.git")
+    message(FATAL_ERROR "google-benchmark is not checked out; make sure to run\n    git submodule update --init benchmarks/google-benchmark")
+endif()
+
+set(BENCHMARK_ENABLE_DOXYGEN OFF)
+set(BENCHMARK_ENABLE_INSTALL OFF)
+set(BENCHMARK_ENABLE_TESTING OFF)
+set(BUILD_SHARED_LIBS OFF)
+
+# TRANSITION, GH-2816: on some machines, librt is found, despite it being a unix-only library
+set(HAVE_LIB_RT OFF)
+
+include_directories(BEFORE "${CMAKE_BINARY_DIR}/out/inc")
+link_directories(BEFORE "${STL_LIBRARY_OUTPUT_DIRECTORY}")
+
+add_subdirectory(google-benchmark EXCLUDE_FROM_ALL)
+
+set(benchmark_headers
+    "inc/udt.hpp"
+    "inc/utility.hpp"
+    "inc/xoshiro.hpp"
+)
+
+function(add_benchmark name)
+    cmake_parse_arguments(PARSE_ARGV 1 "arg" "" "CXX_STANDARD" "")
+
+    if(NOT DEFINED arg_CXX_STANDARD)
+        set(arg_CXX_STANDARD 23)
+    elseif(NOT arg_CXX_STANDARD MATCHES "^[0-9][0-9]$")
+        message(FATAL_ERROR "Unexpected value for CXX_STANDARD: ${arg_CXX_STANDARD}")
+    endif()
+
+    if(NOT DEFINED arg_UNPARSED_ARGUMENTS)
+        message(FATAL_ERROR "benchmark ${name} does not have any source files")
+    endif()
+
+    add_executable(benchmark-${name}
+        ${benchmark_headers}
+        ${arg_UNPARSED_ARGUMENTS}
+    )
+
+    target_compile_features(benchmark-${name} PRIVATE cxx_std_${arg_CXX_STANDARD})
+    target_include_directories(benchmark-${name} PRIVATE inc)
+    target_link_libraries(benchmark-${name} PRIVATE benchmark::benchmark)
+endfunction()
+
+add_benchmark(std_copy
+    src/std_copy.cpp
+    CXX_STANDARD 23
+)

benchmarks/inc/udt.hpp

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+template <typename Contained>
+struct aggregate {
+    Contained c;
+
+    friend bool operator==(const aggregate&, const aggregate&) = default;
+};
+
+template <typename Contained>
+struct non_trivial {
+    Contained c;
+    non_trivial() : c() {}
+    non_trivial(const Contained& src) : c(src) {}
+    non_trivial(const non_trivial& other) : c(other.c) {}
+    non_trivial& operator=(const non_trivial& other) {
+        c = other.c;
+        return *this;
+    }
+    ~non_trivial() {}
+
+    friend bool operator==(const non_trivial&, const non_trivial&) = default;
+};