From e18c8f8e9573882185d709d6b4180d4668d216c2 Mon Sep 17 00:00:00 2001
From: ZhouyihaiDing <ddyihai@google.com>
Date: Fri, 27 Apr 2018 13:17:37 -0700
Subject: [PATCH] Init benchmark tests

---
 tests/qps/Logging/LoggingClientCrossTests.php | 88 +++++++++++++++++++
 tests/qps/Logging/LoggingClientGRPC.php       | 60 +++++++++++++
 tests/qps/Logging/LoggingClientREST.php       | 60 +++++++++++++
 tests/qps/benchmark.php                       | 11 +++
 tests/qps/composer.json                       |  5 ++
 tests/qps/run_test.sh                         |  8 ++
 6 files changed, 232 insertions(+)
 create mode 100644 tests/qps/Logging/LoggingClientCrossTests.php
 create mode 100644 tests/qps/Logging/LoggingClientGRPC.php
 create mode 100644 tests/qps/Logging/LoggingClientREST.php
 create mode 100644 tests/qps/benchmark.php
 create mode 100644 tests/qps/composer.json
 create mode 100644 tests/qps/run_test.sh

diff --git a/tests/qps/Logging/LoggingClientCrossTests.php b/tests/qps/Logging/LoggingClientCrossTests.php
new file mode 100644
index 0000000000..41c3842336
--- /dev/null
+++ b/tests/qps/Logging/LoggingClientCrossTests.php
@@ -0,0 +1,88 @@
+<?php
+
+require dirname(__FILE__).'/../vendor/autoload.php';
+require dirname(__FILE__).'/../benchmark.php';
+use Google\Cloud\Logging\LoggingClient;
+
+function qps_client_main($arg_warm_up, $arg_benchmark)
+{
+    // Disable the batch so each request stands for a RPC
+    $grpcLogger = LoggingClient::psrBatchLogger(
+        'perf-gRPC',
+        [
+            'clientConfig' => [
+                'transport' => 'grpc'
+            ],
+            'batchOptions' => ['batchSize' => 1]
+        ]
+    );
+
+    $restLogger = LoggingClient::psrBatchLogger(
+        'perf-rest',
+        [
+            'clientConfig' => [
+                'transport' => 'rest'
+            ],
+            'batchOptions' => ['batchSize' => 1]
+        ]
+    );
+
+    $warm_up_time = $arg_warm_up;
+    $benchmark_time = $arg_benchmark;
+
+    $benchmark_count = 0;
+    $rest_latency_array = array();
+    $grpc_latency_array = array();
+
+    // First latency
+    $start_time = microtime(true);
+    $grpcLogger->info('a');
+    $grpc_first_latency = microtime(true) - $start_time;
+
+    $start_time = microtime(true);
+    $restLogger->info('a');
+    $rest_first_latency = microtime(true) - $start_time;
+
+    // Warm up
+    $start_time = microtime(true);
+    echo "Start warmup...". PHP_EOL;
+    while(1) {
+        if (microtime(true) - $start_time > $warm_up_time) {
+            break;
+        }
+        $grpcLogger->info('a');
+        $restLogger->info('a');
+    }
+    // Benchmark
+    echo "Start benchmark...". PHP_EOL;
+    $start_time = microtime(true);
+    while(1) {
+        if (microtime(true) - $start_time > $benchmark_time) {
+            break;
+        }
+        $benchmark_count += 1;
+
+        $grpc_start_time = microtime(true);
+        $grpcLogger->info('b');
+        array_push($grpc_latency_array, microtime(true) - $grpc_start_time);
+
+        $rest_start_time = microtime(true);
+        $restLogger->info('b');
+        array_push($rest_latency_array, microtime(true) - $rest_start_time);
+    }
+
+    echo "gRPC transport qps: ".($benchmark_count/array_sum($grpc_latency_array)).PHP_EOL;
+    echo "gRPC latency for first RPC:". $grpc_first_latency. PHP_EOL;
+    sort($grpc_latency_array);
+    print_stats("gRPC", $grpc_latency_array);
+
+    echo "rest transport qps: ". ($benchmark_count/array_sum($rest_latency_array)). PHP_EOL;
+    echo "rest latency for first RPC:". $rest_first_latency. PHP_EOL;
+    sort($rest_latency_array);
+    print_stats("REST", $rest_latency_array);
+}
+
+$arg_warmup = !empty($argv[1]) ? $argv[1] : 20;
+$arg_benchmark = !empty($argv[2]) ? $argv[2] : 40;
+qps_client_main($arg_warmup, $arg_benchmark);
+
diff --git a/tests/qps/Logging/LoggingClientGRPC.php b/tests/qps/Logging/LoggingClientGRPC.php
new file mode 100644
index 0000000000..f39bdbed55
--- /dev/null
+++ b/tests/qps/Logging/LoggingClientGRPC.php
@@ -0,0 +1,60 @@
+<?php
+
+require dirname(__FILE__).'/../vendor/autoload.php';
+require dirname(__FILE__).'/../benchmark.php';
+use Google\Cloud\Logging\LoggingClient;
+
+function qps_client_main($arg_warm_up, $arg_benchmark)
+{
+    // Disable the batch so each request stands for a RPC
+    $grpcLogger = LoggingClient::psrBatchLogger(
+        'perf-gRPC',
+        [
+            'clientConfig' => [
+                'transport' => 'grpc'
+            ],
+            'batchOptions' => ['batchSize' => 1]
+        ]
+    );
+
+    $warm_up_time = $arg_warm_up;
+    $benchmark_time = $arg_benchmark;
+
+    $grpc_benchmark_count = 0;
+    $grpc_latency_array = array();
+    $grpc_first_latency = 0;
+
+    // gRPC QPS test begin
+    $start_time = microtime(true);
+    $grpcLogger->info('a');
+    $grpc_first_latency = microtime(true) - $start_time;
+    $start_time = microtime(true);
+    echo "gRPC transport starts warmup...". PHP_EOL;
+    while(1) {
+        if (microtime(true) - $start_time > $warm_up_time) {
+            break;
+        }
+        $grpcLogger->info('a');
+    }
+    echo "gRPC transport starts benchmark...". PHP_EOL;
+    $start_time = microtime(true);
+    while(1) {
+        $cur_time = microtime(true);
+        if ($cur_time - $start_time > $benchmark_time) {
+            break;
+        }
+        $grpcLogger->info('b');
+        array_push($grpc_latency_array, microtime(true) - $cur_time);
+        $grpc_benchmark_count += 1;
+    }
+
+    echo "gRPC transport qps: ".($grpc_benchmark_count/$benchmark_time).PHP_EOL;
+    echo "gRPC latency for first RPC:". $grpc_first_latency. PHP_EOL;
+    sort($grpc_latency_array);
+    print_stats("gRPC", $grpc_latency_array);
+}
+
+$arg_warmup = !empty($argv[1]) ? $argv[1] : 20;
+$arg_benchmark = !empty($argv[2]) ? $argv[2] : 40;
+qps_client_main($arg_warmup, $arg_benchmark);
+
diff --git a/tests/qps/Logging/LoggingClientREST.php b/tests/qps/Logging/LoggingClientREST.php
new file mode 100644
index 0000000000..304a1420db
--- /dev/null
+++ b/tests/qps/Logging/LoggingClientREST.php
@@ -0,0 +1,60 @@
+<?php
+
+require dirname(__FILE__).'/../vendor/autoload.php';
+require dirname(__FILE__).'/../benchmark.php';
+use Google\Cloud\Logging\LoggingClient;
+
+function qps_client_main($arg_warm_up, $arg_benchmark)
+{
+    // Disable the batch so each request stands for a RPC
+    $restLogger = LoggingClient::psrBatchLogger(
+        'perf-rest',
+        [
+            'clientConfig' => [
+                'transport' => 'rest'
+            ],
+            'batchOptions' => ['batchSize' => 1]
+        ]
+    );
+
+    $warm_up_time = $arg_warm_up;
+    $benchmark_time = $arg_benchmark;
+
+    $rest_benchmark_count = 0;
+    $rest_latency_array = array();
+    $rest_first_latency = 0;
+
+    // REST QPS test begin
+    $start_time = microtime(true);
+    $restLogger->info('a');
+    $rest_first_latency = microtime(true) - $start_time;
+    $start_time = microtime(true);
+    echo "rest transport starts warmup...". PHP_EOL;
+    while(1) {
+        if (microtime(true) - $start_time > $warm_up_time) {
+            break;
+        }
+        $restLogger->info('a');
+    }
+    echo "rest transport starts benchmark...". PHP_EOL;
+    $start_time = microtime(true);
+    while(1) {
+        $cur_time = microtime(true);
+        if ($cur_time - $start_time > $benchmark_time) {
+            break;
+        }
+        $restLogger->info('b');
+        array_push($rest_latency_array, microtime(true) - $cur_time);
+        $rest_benchmark_count += 1;
+    }
+
+    echo "rest transport qps: ". ($rest_benchmark_count/$benchmark_time). PHP_EOL;
+    echo "rest latency for first RPC:". $rest_first_latency. PHP_EOL;
+    sort($rest_latency_array);
+    print_stats("REST", $rest_latency_array);
+}
+
+$arg_warmup = !empty($argv[1]) ? $argv[1] : 20;
+$arg_benchmark = !empty($argv[2]) ? $argv[2] : 40;
+qps_client_main($arg_warmup, $arg_benchmark);
+
diff --git a/tests/qps/benchmark.php b/tests/qps/benchmark.php
new file mode 100644
index 0000000000..4c4823fcb4
--- /dev/null
+++ b/tests/qps/benchmark.php
@@ -0,0 +1,11 @@
+<?php
+
+function print_stats($name, $stats)
+{
+    echo "stats result for ". $name .PHP_EOL;
+    echo "Ave latency :". array_sum($stats)/max(count($stats), 1). PHP_EOL;
+    echo "50% latency :". $stats[count($stats)*0.5]. PHP_EOL;
+    echo "80% latency :". $stats[count($stats)*0.8]. PHP_EOL;
+    echo "90% latency :". $stats[count($stats)*0.9]. PHP_EOL;
+    echo "99% latency :". $stats[min(count($stats)*0.99, count($stats) - 1)]. PHP_EOL;
+}
diff --git a/tests/qps/composer.json b/tests/qps/composer.json
new file mode 100644
index 0000000000..1feb4d9ab1
--- /dev/null
+++ b/tests/qps/composer.json
@@ -0,0 +1,5 @@
+{
+    "require": {
+        "google/cloud": "^0.56.0"
+    }
+}
diff --git a/tests/qps/run_test.sh b/tests/qps/run_test.sh
new file mode 100644
index 0000000000..e0227acad2
--- /dev/null
+++ b/tests/qps/run_test.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# LoggingClient
+php -d extension=grpc.so -d extension=protobuf.so Logging/LoggingClientGRPC.php 20 40
+php -d extension=protobuf.so Logging/LoggingClientREST.php 20 40
+php -d extension=grpc.so -d extension=protobuf.so Logging/LoggingClientCrossTests.php 40 80
+
+