httpclient.benchmarks.yml

# Example:
# crank --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/httpclient.benchmarks.yml --scenario httpclient-kestrel-get --profile aspnet-perf-lin --variable concurrencyPerHttpClient=200

imports:
  - https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/aspnet.profiles.standard.yml

variables:
    warmup: 15 #seconds
    duration: 15 #seconds
    serverPort: 5000
    httpVersion: "1.1"
    useHttps: false
    numberOfHttpClients: 1
    concurrencyPerHttpClient: 1 # unsupported by WRK -- other values will throw
    collectRequestTimings: false # unsupported by WRK -- other values will throw
    responseSize: 0 # 0 for no content in response
    http3StreamLimit: 0 # 0 for server default. Max value is 65535

    # POST-specific parameters:
    requestContentSize: 0 # 0 for no content
    requestContentWriteSize: 81920
    requestContentFlushAfterWrite: false
    requestContentUnknownLength: false

    # HttpClient-specific parameters:
    http11MaxConnectionsPerServer: 0 #unlimited
    http20EnableMultipleConnections: true
    useWinHttpHandler: false
    useHttpMessageInvoker: false
    useDefaultRequestHeaders: false

    # Request headers parameters:
    headersDictionary:
      none: ''
      connectionclose: '--header "Connection: close"'
      expectcontinue: '--header "Expect: 100-continue"' # unsupported by WRK -- will throw
    requestHeaders: none
    generatedStaticRequestHeadersCount: 0
    generatedDynamicRequestHeadersCount: 0

scenarios:
  httpclient-kestrel-get:
    server:
      job: kestrel
      agent: secondary
      variables:
        host: '{{secondaryAddress}}'
    client:
      job: httpClient
      agent: main
      variables:
        scenario: get
        host: '{{secondaryAddress}}'

  httpclient-kestrel-post:
    server:
      job: kestrel
      agent: secondary
      variables:
        host: '{{secondaryAddress}}'
    client:
      job: httpClient
      agent: main
      variables:
        scenario: post
        requestContentSize: 1024
        host: '{{secondaryAddress}}'

  httpclient-kestrel-post-slow:
    server:
      job: kestrel
      agent: secondary
      variables:
        host: '{{secondaryAddress}}'
    client:
      job: httpClient
      agent: main
      variables:
        scenario: post
        requestContentSize: 1024
        requestContentWriteSize: 1
        requestContentFlushAfterWrite: true
        requestContentUnknownLength: true
        host: '{{secondaryAddress}}'

  httpclient-kestrel-configured:
    server:
      job: kestrel
      agent: secondary
      variables:
        httpVersion: "1.1"
        useHttps: true
        responseSize: 8192
        host: '{{secondaryAddress}}'
    client:
      job: httpClient
      agent: main
      variables:
        scenario: get
        httpVersion: "1.1"
        useHttps: true
        numberOfHttpClients: 1
        concurrencyPerHttpClient: '{% if cores > 0 %}{{cores}}{% else %}1{% endif %}'
        http20EnableMultipleConnections: false
        useHttpMessageInvoker: true
        host: '{{secondaryAddress}}'

  wrk-kestrel-get:
    server:
      job: kestrel
      agent: secondary
      variables:
        host: '{{secondaryAddress}}'
    client:
      job: wrk
      agent: main
      variables:
        path: '/get'
        host: '{{secondaryAddress}}'
      onConfigure:
        - '
          if (String(job.Variables.concurrencyPerHttpClient) != "1")
          {
            throw "concurrencyPerHttpClient is unsupported in WRK. Expected: 1, got: " + job.Variables.concurrencyPerHttpClient;
          }
          if (String(job.Variables.collectRequestTimings).toLowerCase() != "false")
          {
            throw "collectRequestTimings is unsupported in WRK. Expected: false, got: " + job.Variables.collectRequestTimings;
          }
          var useHttps = String(job.Variables.useHttps).toLowerCase();
          if (useHttps != "true" && useHttps != "false")
          {
            throw "Unknown value in useHttps. Expected: true or false, got: " + job.Variables.useHttps;
          }
          if (String(job.Variables.requestContentSize) != "0")
          {
            throw "Expected to have requestContentSize=0 for GET scenario, got: " + job.Variables.requestContentSize;
          }
          if (job.Variables.requestHeaders != "none" && job.Variables.requestHeaders != "connectionclose" )
          {
            throw "Unsupported requestHeaders value: " + job.Variables.requestHeaders;
          }
          if (String(job.Variables.generatedStaticRequestHeadersCount) != "0")
          {
            throw "generatedStaticRequestHeadersCount is unsupported in WRK. Expected: 0, got: " + job.Variables.generatedStaticRequestHeadersCount;
          }
          if (String(job.Variables.generatedDynamicRequestHeadersCount) != "0")
          {
            throw "generatedDynamicRequestHeadersCount is unsupported in WRK. Expected: 0, got: " + job.Variables.generatedDynamicRequestHeadersCount;
          }
          '


jobs:
  kestrel:
    source:
      repository: https://github.com/aspnet/benchmarks.git
      branchOrCommit: main
      project: src/BenchmarksApps/HttpClientBenchmarks/Servers/Kestrel/Kestrel.csproj
    readyStateText: Application started # app should write this line to output -- default Kestrel logging does that
    arguments: '--address {{host}} --port {{serverPort}} --useHttps {{useHttps}} --httpVersion {{httpVersion}} --responseSize {{responseSize}} --http3StreamLimit {{http3StreamLimit}}'

  httpClient:
    source:
      repository: https://github.com/aspnet/benchmarks.git
      branchOrCommit: main
      project: src/BenchmarksApps/HttpClientBenchmarks/Clients/HttpClient/HttpClient.csproj
    isConsoleApp: true
    waitForExit: true
    timeout: 1200 #seconds
    arguments: '--address {{host}} --port {{serverPort}} --useHttps {{useHttps}} --path /{{scenario}} --scenario {{scenario}} --httpVersion {{httpVersion}} --numberOfHttpClients {{numberOfHttpClients}} --concurrencyPerHttpClient {{concurrencyPerHttpClient}} --http11MaxConnectionsPerServer {{http11MaxConnectionsPerServer}} --http20EnableMultipleConnections {{http20EnableMultipleConnections}} --useWinHttpHandler {{useWinHttpHandler}} --useHttpMessageInvoker {{useHttpMessageInvoker}} --collectRequestTimings {{collectRequestTimings}} --contentSize {{requestContentSize}} --contentWriteSize {{requestContentWriteSize}} --contentFlushAfterWrite {{requestContentFlushAfterWrite}} --contentUnknownLength {{requestContentUnknownLength}} {{headersDictionary[requestHeaders]}} --generatedStaticHeadersCount {{generatedStaticRequestHeadersCount}} --generatedDynamicHeadersCount {{generatedDynamicRequestHeadersCount}} --useDefaultRequestHeaders {{useDefaultRequestHeaders}} --warmup {{warmup}} --duration {{duration}}'

  wrk:
    source:
      repository: https://github.com/dotnet/crank.git
      branchOrCommit: main
      project: src/Microsoft.Crank.Jobs.Wrk/Microsoft.Crank.Jobs.Wrk.csproj
      sourceKey: wrk
      noBuild: true
    isConsoleApp: true
    waitForExit: true
    arguments: '-c {{numberOfHttpClients}} http{% if useHttps and useHttps != "false" %}s{% endif %}://{{host}}:{{serverPort}}{{path}} --latency -d {{duration}}s -w {{warmup}}s {% if timeout != blank and timeout != empty %} --timeout {{timeout}}s {% endif %} -t {{numberOfHttpClients}} {{headersDictionary[requestHeaders]}}'
    options:
      requiredOperatingSystem: linux