Merge pull request #75599 from ajwerner/ajwerner/fix-rtt-analysis

bench/rttanalysis: skip the first iteration, take average
cockroachdb · Jan 27, 2022 · ba5c832 · ba5c832
2 parents b3ce2b7 + 96c87bb
commit ba5c832
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 13 deletions.
diff --git a/pkg/bench/rttanalysis/rtt_analysis_bench.go b/pkg/bench/rttanalysis/rtt_analysis_bench.go
@@ -108,6 +108,7 @@ func executeRoundTripTest(b testingB, tc RoundTripBenchTestCase, cc ClusterConst
 	defer cluster.close()
 
 	sql := sqlutils.MakeSQLRunner(cluster.conn())
+
 	expData := readExpectationsFile(b)
 
 	exp, haveExp := expData.find(strings.TrimPrefix(b.Name(), "Benchmark"))
@@ -116,7 +117,10 @@ func executeRoundTripTest(b testingB, tc RoundTripBenchTestCase, cc ClusterConst
 	b.ResetTimer()
 	b.StopTimer()
 	var r tracing.Recording
-	for i := 0; i < b.N(); i++ {
+
+	// Do an extra iteration and don't record it in order to deal with effects of
+	// running it the first time.
+	for i := 0; i < b.N()+1; i++ {
 		sql.Exec(b, "CREATE DATABASE bench;")
 		sql.Exec(b, tc.Setup)
 		cluster.clearStatementTrace(tc.Stmt)
@@ -138,7 +142,7 @@ func executeRoundTripTest(b testingB, tc RoundTripBenchTestCase, cc ClusterConst
 		rt, hasRetry := countKvBatchRequestsInRecording(r)
 		if hasRetry {
 			i--
-		} else {
+		} else if i > 0 { // skip the initial iteration
 			roundTrips += rt
 		}
 

diff --git a/pkg/bench/rttanalysis/testdata/benchmark_expectations b/pkg/bench/rttanalysis/testdata/benchmark_expectations
@@ -31,22 +31,22 @@ exp,benchmark
 19,CreateRole/create_role_with_2_options
 20,CreateRole/create_role_with_3_options
 18-19,CreateRole/create_role_with_no_options
-24,DropDatabase/drop_database_0_tables
-30-35,DropDatabase/drop_database_1_table
-36-42,DropDatabase/drop_database_2_tables
-42-47,DropDatabase/drop_database_3_tables
+22,DropDatabase/drop_database_0_tables
+28,DropDatabase/drop_database_1_table
+34,DropDatabase/drop_database_2_tables
+40,DropDatabase/drop_database_3_tables
 25,DropRole/drop_1_role
 32,DropRole/drop_2_roles
 39,DropRole/drop_3_roles
-22,DropSequence/drop_1_sequence
-28,DropSequence/drop_2_sequences
-34,DropSequence/drop_3_sequences
+19,DropSequence/drop_1_sequence
+25,DropSequence/drop_2_sequences
+31,DropSequence/drop_3_sequences
 20-24,DropTable/drop_1_table
 27-31,DropTable/drop_2_tables
 34-38,DropTable/drop_3_tables
 22-25,DropView/drop_1_view
 29-32,DropView/drop_2_views
-39,DropView/drop_3_views
+36,DropView/drop_3_views
 19,Grant/grant_all_on_1_table
 21-22,Grant/grant_all_on_2_tables
 23-24,Grant/grant_all_on_3_tables
@@ -63,8 +63,8 @@ exp,benchmark
 15,ORMQueries/information_schema._pg_index_position
 2,ORMQueries/pg_attribute
 2,ORMQueries/pg_class
-9,ORMQueries/pg_is_other_temp_schema
-21,ORMQueries/pg_is_other_temp_schema_multiple_times
+11,ORMQueries/pg_is_other_temp_schema
+23,ORMQueries/pg_is_other_temp_schema_multiple_times
 4,ORMQueries/pg_my_temp_schema
 4,ORMQueries/pg_my_temp_schema_multiple_times
 4,ORMQueries/pg_namespace

diff --git a/pkg/bench/rttanalysis/testing_shims.go b/pkg/bench/rttanalysis/testing_shims.go
@@ -66,7 +66,7 @@ func (ts tShim) logScope() (getDirectory func() string, close func()) {
 func (ts tShim) GetDirectory() string {
 	return ts.scope.GetDirectory()
 }
-func (ts tShim) N() int      { return 1 }
+func (ts tShim) N() int      { return 2 }
 func (ts tShim) ResetTimer() {}
 func (ts tShim) StopTimer()  {}
 func (ts tShim) StartTimer() {}