feat(minor): Support swiftc option passthrough (#262)

Add support for passthrough of flags to swiftc, e.g.:

```
swift package benchmark --Xswiftc lto=llvm-full --Xswiftc experimental-hermetic-seal-at-link
```

Plugins/BenchmarkCommandPlugin/BenchmarkCommandPlugin.swift

@@ -46,6 +46,7 @@ import PackagePlugin
         let metricsToUse = argumentExtractor.extractOption(named: "metric")
         let debug = argumentExtractor.extractFlag(named: "debug")
         let scale = argumentExtractor.extractFlag(named: "scale")
+        let otherSwiftFlagsSpecified = argumentExtractor.extractOption(named: "Xswiftc")
         var outputFormat: OutputFormat = .text
         var grouping = "benchmark"
         var exportPath = "."
@@ -279,9 +280,13 @@ import PackagePlugin
                 }
             }
 
+            var buildParameters = PackageManager.BuildParameters(configuration: .release)
+
+            buildParameters.otherSwiftcFlags.append(contentsOf: otherSwiftFlagsSpecified.map { "-\($0)" })
+
             let buildResult = try packageManager.build(
                 .product(benchmarkToolModule.name),
-                parameters: .init(configuration: .release)
+                parameters: buildParameters
             )
 
             guard buildResult.succeeded else {

Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift

@@ -11,21 +11,21 @@
 let help =
     """
     OVERVIEW: Run benchmarks or update, compare or check performance baselines
-
+    
     Performs operations on benchmarks (running or listing them), as well as storing, comparing baselines as well as checking them for threshold deviations.
-
+    
     The init command will create a skeleton benchmark suite for you and add it to Package.swift.
-
+    
     For the 'text' default format, the output is implicitly 'stdout' unless otherwise specified.
     For all other formats, the output is to a file in either the current working directory, or
     the directory specified by the '--path' option, unless the special 'stdout' path is specified
     in which case output will go to stdout (useful for e.g. baseline 'tsv' format export piped to youplot).
-
+    
     To allow writing to the package directory, you may need to pass the appropriate option to swift package:
     swift package --allow-writing-to-package-directory benchmark <command> <options>
-
+    
     USAGE: swift package benchmark <command>
-
+    
        swift package benchmark [run] <options>
        swift package benchmark init <benchmarkTargetName>
        swift package benchmark list
@@ -36,23 +36,22 @@ let help =
        swift package benchmark baseline check <baseline> [<otherBaseline>] [<options>]
        swift package benchmark baseline compare <baseline> [<otherBaseline>] [<options>]
        swift package benchmark help
-
+    
     ARGUMENTS:
-    <command>               The benchmark command to perform, one of: ["run", "list", "baseline", "help", "init"]. If not specified, 'run' is implied.
-
+    <command>               The benchmark command to perform. If not specified, 'run' is implied. (values: run, list, baseline, help, init)
+    
     OPTIONS:
     --filter <filter>       Benchmarks matching the regexp filter that should be run
     --skip <skip>           Benchmarks matching the regexp filter that should be skipped
     --target <target>       Benchmark targets matching the regexp filter that should be run
     --skip-target <skip-target>
                           Benchmark targets matching the regexp filter that should be skipped
-    --format <format>       The output format to use, one of: ["text", "markdown", "influx", "jmh", "histogramEncoded", "histogram", "histogramSamples", "histogramPercentiles", "metricP90AbsoluteThresholds"], default is 'text'
-    --metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks, valid values are:
-                          ["cpuUser", "cpuSystem", "cpuTotal", "wallClock", "throughput", "peakMemoryResident", "peakMemoryResidentDelta", "peakMemoryVirtual",
-                          "mallocCountSmall", "mallocCountLarge", "mallocCountTotal", "allocatedResidentMemory", "memoryLeaked", "syscalls", "contextSwitches",
-                          "threads", "threadsRunning", "readSyscalls", "writeSyscalls", "readBytesLogical", "writeBytesLogical", "readBytesPhysical",
-                          "writeBytesPhysical", "instructions", "retainCount", "releaseCount", "retainReleaseDelta", "custom"]
-    --path <path>           The path where exported data is stored, default is the current directory (".").
+    --format <format>       The output format to use, default is 'text' (values: text, markdown, influx, jmh, histogramEncoded, histogram, histogramSamples, histogramPercentiles, metricP90AbsoluteThresholds)
+    --metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks. (values: cpuUser, cpuSystem, cpuTotal, wallClock, throughput,
+                          peakMemoryResident, peakMemoryResidentDelta, peakMemoryVirtual, mallocCountSmall, mallocCountLarge, mallocCountTotal, allocatedResidentMemory, memoryLeaked, syscalls, contextSwitches,
+                          threads, threadsRunning, readSyscalls, writeSyscalls, readBytesLogical, writeBytesLogical, readBytesPhysical, writeBytesPhysical, instructions, retainCount, releaseCount,
+                          retainReleaseDelta, custom)
+    --path <path>           The path where exported data is stored, default is the current directory ("."). 
     --quiet                 Specifies that output should be suppressed (useful for if you just want to check return code)
     --scale                 Specifies that some of the text output should be scaled using the scalingFactor (denoted by '*' in output)
     --check-absolute        Set to true if thresholds should be checked against an absolute reference point rather than delta between baselines.
@@ -66,6 +65,7 @@ let help =
                           The path from which p90 thresholds will be loaded for absolute threshold checks.
                           This implicitly sets --check-absolute to true as well.
     --no-progress           Specifies that benchmark progress information should not be displayed
-    --grouping <grouping>   The grouping to use, one of: ["metric", "benchmark"]. default is 'benchmark'
+    --grouping <grouping>   The grouping to use, one of: ["metric", "benchmark"]. default is 'benchmark' (values: metric, benchmark)
+    --xswiftc <xswiftc>     Pass an argument to the Swift compiler when building the benchmark
     -h, --help              Show help information.
     """

Plugins/BenchmarkHelpGenerator/BenchmarkHelpGenerator.swift

@@ -82,7 +82,7 @@ struct Benchmark: AsyncParsableCommand {
         """
     )
 
-    @Argument(help: "The benchmark command to perform, one of: \((Command.allCases).map { String(describing: $0) }). If not specified, 'run' is implied.")
+    @Argument(help: "The benchmark command to perform. If not specified, 'run' is implied.")
     var command: Command
 
     @Option(name: .long, help: "Benchmarks matching the regexp filter that should be run")
@@ -97,10 +97,10 @@ struct Benchmark: AsyncParsableCommand {
     @Option(name: .long, help: "Benchmark targets matching the regexp filter that should be skipped")
     var skipTarget: [String] = []
 
-    @Option(name: .long, help: "The output format to use, one of: \((OutputFormat.allCases).map { String(describing: $0) }), default is '\(OutputFormat.text.rawValue)'")
+    @Option(name: .long, help: "The output format to use, default is '\(OutputFormat.text.rawValue)'")
     var format: OutputFormat
 
-    @Option(name: .long, help: "Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks, valid values are: \(availableMetrics)")
+    @Option(name: .long, help: "Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks. (values: \(availableMetrics.joined(separator: ", ")))")
     var metric: [String] = []
 
     @Option(name: .long, help: "The path where exported data is stored, default is the current directory (\".\"). ")
@@ -137,5 +137,8 @@ struct Benchmark: AsyncParsableCommand {
     @Option(name: .long, help: "The grouping to use, one of: \((Grouping.allCases).map { String(describing: $0) }). default is '\(Grouping.benchmark.rawValue)'")
     var grouping: Grouping
 
+    @Option(name: .long, help: "Pass an argument to the Swift compiler when building the benchmark")
+    var Xswiftc: String
+
     mutating func run() async throws {}
 }

Sources/Benchmark/Documentation.docc/RunningBenchmarks.md

@@ -75,21 +75,20 @@ swift package benchmark baseline compare <baseline> [<otherBaseline>] [<options>
 swift package benchmark help
 
 ARGUMENTS:
-<command>               The benchmark command to perform, one of: ["run", "list", "baseline", "help", "init"]. If not specified, 'run' is implied.
+<command>               The benchmark command to perform. If not specified, 'run' is implied. (values: run, list, baseline, help, init)
 
 OPTIONS:
 --filter <filter>       Benchmarks matching the regexp filter that should be run
 --skip <skip>           Benchmarks matching the regexp filter that should be skipped
 --target <target>       Benchmark targets matching the regexp filter that should be run
 --skip-target <skip-target>
 Benchmark targets matching the regexp filter that should be skipped
---format <format>       The output format to use, one of: ["text", "markdown", "influx", "jmh", "histogramEncoded", "histogram", "histogramSamples", "histogramPercentiles", "metricP90AbsoluteThresholds"], default is 'text'
---metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks, valid values are:
-["cpuUser", "cpuSystem", "cpuTotal", "wallClock", "throughput", "peakMemoryResident", "peakMemoryResidentDelta", "peakMemoryVirtual",
-"mallocCountSmall", "mallocCountLarge", "mallocCountTotal", "allocatedResidentMemory", "memoryLeaked", "syscalls", "contextSwitches",
-"threads", "threadsRunning", "readSyscalls", "writeSyscalls", "readBytesLogical", "writeBytesLogical", "readBytesPhysical",
-"writeBytesPhysical", "instructions", "retainCount", "releaseCount", "retainReleaseDelta", "custom"]
---path <path>           The path where exported data is stored, default is the current directory (".").
+--format <format>       The output format to use, default is 'text' (values: text, markdown, influx, jmh, histogramEncoded, histogram, histogramSamples, histogramPercentiles, metricP90AbsoluteThresholds)
+--metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks. (values: cpuUser, cpuSystem, cpuTotal, wallClock, throughput,
+peakMemoryResident, peakMemoryResidentDelta, peakMemoryVirtual, mallocCountSmall, mallocCountLarge, mallocCountTotal, allocatedResidentMemory, memoryLeaked, syscalls, contextSwitches,
+threads, threadsRunning, readSyscalls, writeSyscalls, readBytesLogical, writeBytesLogical, readBytesPhysical, writeBytesPhysical, instructions, retainCount, releaseCount,
+retainReleaseDelta, custom)
+--path <path>           The path where exported data is stored, default is the current directory ("."). 
 --quiet                 Specifies that output should be suppressed (useful for if you just want to check return code)
 --scale                 Specifies that some of the text output should be scaled using the scalingFactor (denoted by '*' in output)
 --check-absolute        Set to true if thresholds should be checked against an absolute reference point rather than delta between baselines.
@@ -103,7 +102,8 @@ By default, thresholds are checked comparing two baselines, or a baseline and a
 The path from which p90 thresholds will be loaded for absolute threshold checks.
 This implicitly sets --check-absolute to true as well.
 --no-progress           Specifies that benchmark progress information should not be displayed
---grouping <grouping>   The grouping to use, one of: ["metric", "benchmark"]. default is 'benchmark'
+--grouping <grouping>   The grouping to use, one of: ["metric", "benchmark"]. default is 'benchmark' (values: metric, benchmark)
+--xswiftc <xswiftc>     Pass an argument to the Swift compiler when building the benchmark
 -h, --help              Show help information.
 ```
 
@@ -182,6 +182,14 @@ swift package --disable-sandbox benchmark
 
 This is also required for e.g. benchmarks that uses the network.
 
+## Specifying specific flags to swiftc
+
+It is possible to pass arbitrary flags to swiftc using the `Xswiftc` option, e.g.:
+
+```
+swift package benchmark --Xswiftc lto=llvm-full --Xswiftc experimental-hermetic-seal-at-link
+```
+
 ## Sample usage
 
 ### Run all benchmark targets: