Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Init benchmark tests #10

Merged
merged 8 commits into from
May 29, 2018
Merged

Init benchmark tests #10

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c7c9f06
Init benchmark tests
ZhouyihaiDing Apr 27, 2018
3efbc05
update measuring the first RPC including initializing the client
ZhouyihaiDing May 1, 2018
03ed050
update composer.json
ZhouyihaiDing May 1, 2018
e414f96
add payload
ZhouyihaiDing May 2, 2018
2c2c70f
add ext-curl
ZhouyihaiDing May 2, 2018
a831f7d
Write different logs. fix missing ;
ZhouyihaiDing May 2, 2018
8ec2bdf
remove some tests, only leave cross test
ZhouyihaiDing May 2, 2018
b1ce3d2
different payload should also be measured crossly; 1k as default
ZhouyihaiDing May 3, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions tests/qps/Logging/LoggingClientCrossTests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
<?php

require dirname(__FILE__).'/../vendor/autoload.php';
require dirname(__FILE__).'/../benchmark.php';
use Google\Cloud\Logging\LoggingClient;

function qps_client_main($arg_warm_up, $arg_benchmark)
{
// Disable the batch so each request stands for a RPC
$grpcLogger = LoggingClient::psrBatchLogger(
'perf-gRPC',
[
'clientConfig' => [
'transport' => 'grpc'
],
'batchOptions' => ['batchSize' => 1]
]
);

$restLogger = LoggingClient::psrBatchLogger(
'perf-rest',
[
'clientConfig' => [
'transport' => 'rest'
],
'batchOptions' => ['batchSize' => 1]
]
);

$warm_up_time = $arg_warm_up;
$benchmark_time = $arg_benchmark;

$benchmark_count = 0;
$rest_latency_array = array();
$grpc_latency_array = array();

// First latency
$start_time = microtime(true);
$grpcLogger->info('a');
$grpc_first_latency = microtime(true) - $start_time;

$start_time = microtime(true);
$restLogger->info('a');
$rest_first_latency = microtime(true) - $start_time;

// Warm up
$start_time = microtime(true);
echo "Start warmup...". PHP_EOL;
while(1) {
if (microtime(true) - $start_time > $warm_up_time) {
break;
}
$grpcLogger->info('a');
$restLogger->info('a');
}
// Benchmark
echo "Start benchmark...". PHP_EOL;
$start_time = microtime(true);
while(1) {
if (microtime(true) - $start_time > $benchmark_time) {
break;
}
$benchmark_count += 1;

$grpc_start_time = microtime(true);
$grpcLogger->info('b');
array_push($grpc_latency_array, microtime(true) - $grpc_start_time);

$rest_start_time = microtime(true);
$restLogger->info('b');
array_push($rest_latency_array, microtime(true) - $rest_start_time);
}

echo "gRPC transport qps: ".($benchmark_count/array_sum($grpc_latency_array)).PHP_EOL;
echo "gRPC latency for first RPC:". $grpc_first_latency. PHP_EOL;
sort($grpc_latency_array);
print_stats("gRPC", $grpc_latency_array);

echo "rest transport qps: ". ($benchmark_count/array_sum($rest_latency_array)). PHP_EOL;
echo "rest latency for first RPC:". $rest_first_latency. PHP_EOL;
sort($rest_latency_array);
print_stats("REST", $rest_latency_array);
}

$arg_warmup = !empty($argv[1]) ? $argv[1] : 20;
$arg_benchmark = !empty($argv[2]) ? $argv[2] : 40;
qps_client_main($arg_warmup, $arg_benchmark);

60 changes: 60 additions & 0 deletions tests/qps/Logging/LoggingClientGRPC.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
<?php

require dirname(__FILE__).'/../vendor/autoload.php';
require dirname(__FILE__).'/../benchmark.php';
use Google\Cloud\Logging\LoggingClient;

function qps_client_main($arg_warm_up, $arg_benchmark)
{
// Disable the batch so each request stands for a RPC
$grpcLogger = LoggingClient::psrBatchLogger(
'perf-gRPC',
[
'clientConfig' => [
'transport' => 'grpc'
],
'batchOptions' => ['batchSize' => 1]
]
);

$warm_up_time = $arg_warm_up;
$benchmark_time = $arg_benchmark;

$grpc_benchmark_count = 0;
$grpc_latency_array = array();
$grpc_first_latency = 0;

// gRPC QPS test begin
$start_time = microtime(true);
$grpcLogger->info('a');
$grpc_first_latency = microtime(true) - $start_time;
$start_time = microtime(true);
echo "gRPC transport starts warmup...". PHP_EOL;
while(1) {
if (microtime(true) - $start_time > $warm_up_time) {
break;
}
$grpcLogger->info('a');
}
echo "gRPC transport starts benchmark...". PHP_EOL;
$start_time = microtime(true);
while(1) {
$cur_time = microtime(true);
if ($cur_time - $start_time > $benchmark_time) {
break;
}
$grpcLogger->info('b');
array_push($grpc_latency_array, microtime(true) - $cur_time);
$grpc_benchmark_count += 1;
}

echo "gRPC transport qps: ".($grpc_benchmark_count/$benchmark_time).PHP_EOL;
echo "gRPC latency for first RPC:". $grpc_first_latency. PHP_EOL;
sort($grpc_latency_array);
print_stats("gRPC", $grpc_latency_array);
}

$arg_warmup = !empty($argv[1]) ? $argv[1] : 20;
$arg_benchmark = !empty($argv[2]) ? $argv[2] : 40;
qps_client_main($arg_warmup, $arg_benchmark);

60 changes: 60 additions & 0 deletions tests/qps/Logging/LoggingClientREST.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
<?php

require dirname(__FILE__).'/../vendor/autoload.php';
require dirname(__FILE__).'/../benchmark.php';
use Google\Cloud\Logging\LoggingClient;

function qps_client_main($arg_warm_up, $arg_benchmark)
{
// Disable the batch so each request stands for a RPC
$restLogger = LoggingClient::psrBatchLogger(
'perf-rest',
[
'clientConfig' => [
'transport' => 'rest'
],
'batchOptions' => ['batchSize' => 1]
]
);

$warm_up_time = $arg_warm_up;
$benchmark_time = $arg_benchmark;

$rest_benchmark_count = 0;
$rest_latency_array = array();
$rest_first_latency = 0;

// REST QPS test begin
$start_time = microtime(true);
$restLogger->info('a');
$rest_first_latency = microtime(true) - $start_time;
$start_time = microtime(true);
echo "rest transport starts warmup...". PHP_EOL;
while(1) {
if (microtime(true) - $start_time > $warm_up_time) {
break;
}
$restLogger->info('a');
}
echo "rest transport starts benchmark...". PHP_EOL;
$start_time = microtime(true);
while(1) {
$cur_time = microtime(true);
if ($cur_time - $start_time > $benchmark_time) {
break;
}
$restLogger->info('b');
array_push($rest_latency_array, microtime(true) - $cur_time);
$rest_benchmark_count += 1;
}

echo "rest transport qps: ". ($rest_benchmark_count/$benchmark_time). PHP_EOL;
echo "rest latency for first RPC:". $rest_first_latency. PHP_EOL;
sort($rest_latency_array);
print_stats("REST", $rest_latency_array);
}

$arg_warmup = !empty($argv[1]) ? $argv[1] : 20;
$arg_benchmark = !empty($argv[2]) ? $argv[2] : 40;
qps_client_main($arg_warmup, $arg_benchmark);

22 changes: 22 additions & 0 deletions tests/qps/Logging/logging_grpc.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?php
use Google\Cloud\Logging\Logger;
use Google\Cloud\Logging\LoggingClient;
putenv("GOOGLE_APPLICATION_CREDENTIALS=./grpcwebtesting_key.json");
require_once './vendor/autoload.php';

$grpcLogger = LoggingClient::psrBatchLogger(
'perf-grpc',
[
'clientConfig' => [
'transport' => 'grpc'
],
'batchOptions' => ['batchSize' => 1]
]
);

$start_time = microtime(true);
$grpcLogger->info('s');
$first_latency = microtime(true) - $start_time;
echo "rest latency for first RPC:". $first_latency. PHP_EOL;
file_put_contents('logs_grpc.txt', $first_latency.PHP_EOL , FILE_APPEND | LOCK_EX);

21 changes: 21 additions & 0 deletions tests/qps/Logging/logging_rest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<?php
use Google\Cloud\Logging\Logger;
use Google\Cloud\Logging\LoggingClient;
putenv("GOOGLE_APPLICATION_CREDENTIALS=./grpcwebtesting_key.json");
require_once './vendor/autoload.php';

$restLogger = LoggingClient::psrBatchLogger(
'perf-rest',
[
'clientConfig' => [
'transport' => 'rest'
],
'batchOptions' => ['batchSize' => 1]
]
);

$start_time = microtime(true);
$restLogger->info('e');
$first_latency = microtime(true) - $start_time;
echo "rest latency for first RPC:". $first_latency. PHP_EOL;
file_put_contents('logs_rest.txt', $first_latency.PHP_EOL , FILE_APPEND | LOCK_EX);
11 changes: 11 additions & 0 deletions tests/qps/benchmark.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?php

function print_stats($name, $stats)
{
echo "stats result for ". $name .PHP_EOL;
echo "Ave latency :". array_sum($stats)/max(count($stats), 1). PHP_EOL;
echo "50% latency :". $stats[count($stats)*0.5]. PHP_EOL;
echo "80% latency :". $stats[count($stats)*0.8]. PHP_EOL;
echo "90% latency :". $stats[count($stats)*0.9]. PHP_EOL;
echo "99% latency :". $stats[min(count($stats)*0.99, count($stats) - 1)]. PHP_EOL;
}
5 changes: 5 additions & 0 deletions tests/qps/composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"require": {
"google/cloud": "^0.56.0"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make sense to have these requirements?

  "ext-json": "*",
  "ext-protobuf": "*"

or something like that?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done. Thanks!

}
}
8 changes: 8 additions & 0 deletions tests/qps/run_fpm_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash

# LoggingClient
for i in {1..100}
do
curl -I localhost/logging_rest.php
curl -I localhost/logging_grpc.php
done
8 changes: 8 additions & 0 deletions tests/qps/run_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash

# LoggingClient
php -d extension=grpc.so -d extension=protobuf.so Logging/LoggingClientGRPC.php 20 40
php -d extension=protobuf.so Logging/LoggingClientREST.php 20 40
php -d extension=grpc.so -d extension=protobuf.so Logging/LoggingClientCrossTests.php 40 80