Make max_duration configurable in task configuration (#807)

Make max_duration configurable in mlperf_task.proto
mlcommons · Nov 14, 2023 · e02bdc8 · e02bdc8
1 parent 2cdc849
commit e02bdc8
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 7 deletions.
diff --git a/docs/result-spec.md b/docs/result-spec.md
@@ -101,6 +101,8 @@ If you enable Submission mode, both `performance_run` and `accuracy_run` values
   May be null if accuracy was not tested in this benchmark.
 * `min_duration`: floating point number  
   Value from `task.min_duration` for this benchmark from selected tasks.pbtxt file.
+* `max_duration`: floating point number  
+  Value from `task.max_duration` for this benchmark from selected tasks.pbtxt file.
 * `min_samples`: integer number  
   Value from `task.min_query_count` for this benchmark from selected tasks.pbtxt file.
 * `backend_info`: map

diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt
@@ -6,6 +6,7 @@ task {
   name: "Image Classification"
   min_query_count: 1024
   min_duration: 60
+  max_duration: 600
   max_throughput: 1000
   max_accuracy: 1.0
   scenario: "SingleStream"
@@ -42,6 +43,7 @@ task {
   name: "Object Detection"
   min_query_count: 1024
   min_duration: 60
+  max_duration: 600
   max_throughput: 2000
   max_accuracy: 1.0
   scenario: "SingleStream"
@@ -78,6 +80,7 @@ task {
   name: "v2.0 Image Segmentation"
   min_query_count: 1024
   min_duration: 60
+  max_duration: 600
   max_throughput: 2000
   max_accuracy: 1.0
   scenario: "SingleStream"
@@ -113,6 +116,7 @@ task {
   name: "Language Understanding"
   min_query_count: 1024
   min_duration: 60
+  max_duration: 600
   max_throughput: 2000
   max_accuracy: 1.0
   scenario: "SingleStream"
@@ -145,6 +149,7 @@ task {
   name: "Super Resolution "
   min_query_count: 1024
   min_duration: 60
+  max_duration: 600
   max_throughput: 2000
   max_accuracy: 1.0
   scenario: "SingleStream"
@@ -179,6 +184,7 @@ task {
   name: "Image Classification (Offline)"
   min_query_count: 24576
   min_duration: 0
+  max_duration: 0
   max_throughput: 2000
   max_accuracy: 1.0
   scenario: "Offline"

diff --git a/flutter/cpp/proto/mlperf_task.proto b/flutter/cpp/proto/mlperf_task.proto
@@ -31,7 +31,7 @@ message MLPerfConfig {
 // Config of the mlperf tasks.
 // A task is basically a combination of models and a dataset.
 //
-// Next ID: 10
+// Next ID: 11
 message TaskConfig {
   // Must be unique in one task file. Ex: image_classification
   // used to match backend settings
@@ -41,7 +41,9 @@ message TaskConfig {
   // Minimum number of samples the test should run in the performance mode.
   required int32 min_query_count = 3;
   // Minimum duration the test should run in the performance mode, in seconds.
-  required double min_duration = 4;
+  required double min_duration = 4 [default = 60];
+  // Maximum duration the test should run in the performance mode, in seconds.
+  required double max_duration = 10 [default = 600];
   // Max expected throughput score
   required float max_throughput = 5;
   // Max expected accuracy

diff --git a/flutter/lib/app_constants.dart b/flutter/lib/app_constants.dart
@@ -108,7 +108,3 @@ class BackendId {
     apple,
   ];
 }
-
-class BenchmarkSettings {
-  static const double maxDuration = 10 * 60; // 10 minutes
-}
diff --git a/flutter/lib/benchmark/benchmark.dart b/flutter/lib/benchmark/benchmark.dart
@@ -73,7 +73,6 @@ class Benchmark {
     final dataset = runMode.chooseDataset(taskConfig);
 
     int minQueryCount;
-    double maxDuration = BenchmarkSettings.maxDuration;
     double minDuration;
     if (testMinDuration != 0) {
       minQueryCount = testMinQueryCount;
@@ -85,6 +84,7 @@ class Benchmark {
       minQueryCount = taskConfig.minQueryCount;
       minDuration = taskConfig.minDuration;
     }
+    double maxDuration = taskConfig.maxDuration;
 
     final settings = pb.SettingList(
       setting: commonSettings,