mlperf_task.proto

/* Copyright 2019 The MLPerf Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
syntax = "proto2";

package mlperf.mobile;

option cc_enable_arenas = true;
option java_multiple_files = true;
option java_package = "org.mlperf.proto";

// App config
//
// Next ID: 2
message MLPerfConfig {
  // All tasks will be run.
  repeated TaskConfig task = 1;
}

// Config of the mlperf tasks.
// A task is basically a combination of models and a dataset.
//
// Next ID: 11
message TaskConfig {
  // Must be unique in one task file. Ex: image_classification
  // used to match backend settings
  required string id = 1;
  // Human-readable name. Ex: Image classification.
  required string name = 2;
  // Minimum number of samples the test should run in the performance mode.
  required int32 min_query_count = 3;
  // Minimum duration the test should run in the performance mode, in seconds.
  required double min_duration = 4 [default = 60];
  // Maximum duration the test should run in the performance mode, in seconds.
  required double max_duration = 10 [default = 600];
  // Max expected throughput score
  required float max_throughput = 5;
  // Max expected accuracy
  required float max_accuracy = 6;
  // LoadGen parameter. Allowed values: SingleStream, Offline
  required string scenario = 7;
  required DatasetConfig datasets = 8;
  required ModelConfig model = 9;
}

// Datasets for a task
//
// Next ID: 5
message DatasetConfig {
  // Type of the dataset.
  enum DatasetType {
    NONE = 0;
    IMAGENET = 1;
    COCO = 2;
    SQUAD = 3;
    ADE20K = 4;
    SNUSR = 5;
  }
  required DatasetType type = 1;
  // Config of the dataset.
  // full is used for Accuracy run.
  required OneDatasetConfig full = 2;
  // lite is used for Performance run.
  required OneDatasetConfig lite = 3;
  // tiny is used for unit test.
  required OneDatasetConfig tiny = 4;
}

// Config of one dataset
//
// Next ID: 4
message OneDatasetConfig {
  // Human-readable name of the dataset
  required string name = 1;
  // URL or local path to dataset input files
  required string input_path = 2;
  // URL or local path to dataset groundtruth files
  required string groundtruth_path = 3;
}

// Config of a model.
//
// Next ID: 7
message ModelConfig {
  // unique ID used by UI to find matching description
  required string id = 1;
  // Human-readable name of the model.
  required string name = 2;
  // Offset value of the model if applicable.
  optional int32 offset = 3 [default = 0];
  // image width of the input (if model takes images)
  optional int32 image_width = 4;
  // image height of the input (if model takes images)
  optional int32 image_height = 5;
  // Number of detection classes if applicable
  optional int32 num_classes = 6;
}