diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 9f300854d..ccf84647c 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -86,10 +86,10 @@ jobs: python3 -m pip install awscli - name: Download Samsung libraries env: - AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }} run: | - aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \ + aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \ cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/samsung_libs_v3.1_20231116.zip /tmp/ && \ unzip /tmp/samsung_libs_v3.1_20231116.zip -d /tmp/samsung_libs_v3.1_20231116 && \ rm /tmp/samsung_libs_v3.1_20231116.zip && \ @@ -97,12 +97,12 @@ jobs: mv /tmp/samsung_libs_v3.1_20231116/* mobile_back_samsung/samsung/lib/internal/ - name: Download QTI libraries env: - AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }} run: | - aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \ + aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \ cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/qaisw-2.13.4.230831213719_60417-linux.zip /tmp/ && \ - unzip -P ${AWS_SECRET_ACCESS_KEY} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \ + unzip -P ${{ secrets.READONLY_AWS_KEY }} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \ rm /tmp/qaisw-2.13.4.230831213719_60417-linux.zip && \ mv /tmp/qaisw-2.13.4.230831213719_60417-linux/qaisw-2.13.4.230831213719_60417 mobile_back_qti/ - name: Cache bazel diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index 01029bfd1..7795e39fd 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -246,9 +246,9 @@ task { model { id: "MobileNetEdgeTPUv2-L" name: "MobileNetEdgeTPUv2-L" - offset: 1 - image_width: 224 - image_height: 224 + offset: 0 + image_width: 384 + image_height: 384 num_classes: 1000 } } @@ -283,9 +283,9 @@ task { model { id: "MobileNetEdgeTPUv2-L" name: "MobileNetEdgeTPUv2-L" - offset: 1 - image_width: 224 - image_height: 224 + offset: 0 + image_width: 384 + image_height: 384 num_classes: 1000 } } diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index 880f07fe4..dc50aee8b 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -28,14 +28,14 @@ const Map _imageClassification = { }; const Map _imageClassificationV2 = { - 'cpu': Interval(min: 0.69, max: 0.71), - 'npu': Interval(min: 0.69, max: 0.71), - 'tpu': Interval(min: 0.69, max: 0.71), - 'ane': Interval(min: 0.69, max: 0.71), - 'cpu&gpu&ane': Interval(min: 0.69, max: 0.71), - 'snpe_dsp': Interval(min: 0.69, max: 0.71), - 'psnpe_dsp': Interval(min: 0.69, max: 0.71), - 'neuron-mdla': Interval(min: 0.69, max: 0.71), + 'cpu': Interval(min: 0.88, max: 0.91), + 'npu': Interval(min: 0.88, max: 0.91), + 'tpu': Interval(min: 0.88, max: 0.91), + 'ane': Interval(min: 0.79, max: 0.91), + 'cpu&gpu&ane': Interval(min: 0.79, max: 0.91), + 'snpe_dsp': Interval(min: 0.88, max: 0.91), + 'psnpe_dsp': Interval(min: 0.88, max: 0.91), + 'neuron-mdla': Interval(min: 0.79, max: 0.81), }; const Map _objectDetection = { diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 29c215a53..85c7a3c59 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -55,24 +55,24 @@ const Map> _imageClassification = { // TODO (anhappdev): update the expected value for _imageClassificationV2 after gathering some statistics const Map> _imageClassificationV2 = { _kTFLiteBackend: { - _kCloudBuildX23: Interval(min: 2, max: 11), - _kCloudBuildX28: Interval(min: 2, max: 12), + _kCloudBuildX23: Interval(min: 1, max: 9), + _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 1, max: 37), - _kPixel5: Interval(min: 60, max: 75), - _kIphoneOnGitHubAction: Interval(min: 1, max: 7), + _kPixel5: Interval(min: 20, max: 75), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), _kIphoneOnMacbookM1: Interval(min: 10, max: 27), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 1, max: 7), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), }, _kPixelBackend: { - _kPixel6: Interval(min: 500, max: 600), + _kPixel6: Interval(min: 100, max: 600), }, _kQtiBackend: { _kS22Ultra: Interval(min: 1900, max: 2200), }, _kMediatekBackend: { - _kDN2103: Interval(min: 30, max: 50), + _kDN2103: Interval(min: 5, max: 50), }, }; @@ -89,7 +89,7 @@ const Map> _objectDetection = { _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4), }, _kPixelBackend: { - _kPixel6: Interval(min: 250, max: 450), + _kPixel6: Interval(min: 200, max: 450), }, _kQtiBackend: { _kS22Ultra: Interval(min: 800, max: 1400), @@ -195,24 +195,24 @@ const Map> _imageClassificationOffline = { // TODO (anhappdev): update the expected value for _imageClassificationOfflineV2 after gathering some statistics const Map> _imageClassificationOfflineV2 = { _kTFLiteBackend: { - _kCloudBuildX23: Interval(min: 4, max: 14), - _kCloudBuildX28: Interval(min: 3, max: 14), + _kCloudBuildX23: Interval(min: 1, max: 9), + _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 20, max: 60), - _kPixel5: Interval(min: 60, max: 180), - _kIphoneOnGitHubAction: Interval(min: 1, max: 15), + _kPixel5: Interval(min: 20, max: 180), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), _kIphoneOnMacbookM1: Interval(min: 10, max: 45), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 1, max: 15), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), }, _kPixelBackend: { - _kPixel6: Interval(min: 700, max: 1700), + _kPixel6: Interval(min: 100, max: 1700), }, _kQtiBackend: { _kS22Ultra: Interval(min: 2600, max: 3000), }, _kMediatekBackend: { - _kDN2103: Interval(min: 75, max: 110), + _kDN2103: Interval(min: 5, max: 90), }, }; diff --git a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt index 7e8389600..758d675cc 100644 --- a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt +++ b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt @@ -65,22 +65,22 @@ benchmark_setting { delegate_name: "CPU & GPU & ANE" accelerator_name: "cpu&gpu&ane" accelerator_desc: "All compute units" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & GPU" accelerator_name: "cpu&gpu" accelerator_desc: "CPU and GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_selected: "CPU & GPU & ANE" } @@ -93,24 +93,24 @@ benchmark_setting { accelerator_name: "cpu&gpu&ane" accelerator_desc: "All compute units" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & GPU" accelerator_name: "cpu&gpu" accelerator_desc: "CPU and GPU" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_selected: "CPU & GPU & ANE" } diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 1015847c5..863d4909b 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -60,11 +60,13 @@ tflite-run-ic-v2: bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification_v2 \ --mode=PerformanceOnly \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-fp32.tflite" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 tflite-run-ic-offline-v2: cd ${REPO_ROOT_DIR} && \ @@ -73,11 +75,13 @@ tflite-run-ic-offline-v2: --scenario=Offline \ --batch_size=32 \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-fp32.tflite" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 tflite-run-od: cd ${REPO_ROOT_DIR} && \ @@ -155,11 +159,13 @@ coreml-run-ic-v2: bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification \ --mode=PerformanceOnly \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 coreml-run-ic-offline-v2: cd ${REPO_ROOT_DIR} && \ @@ -168,11 +174,13 @@ coreml-run-ic-offline-v2: --scenario=Offline \ --batch_size=32 \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 coreml-run-od: cd ${REPO_ROOT_DIR} && \ diff --git a/mobile_back_apple/models/icv2.py b/mobile_back_apple/models/icv2.py index 61ff905e9..4af313776 100644 --- a/mobile_back_apple/models/icv2.py +++ b/mobile_back_apple/models/icv2.py @@ -16,36 +16,34 @@ # limitations under the License. import coremltools as ct -import tensorflow as tf -import tensorflow_hub as hub def main(): - print("Retrieve model from TFHub...") - keras_layer = hub.KerasLayer('https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1') - keras_model = tf.keras.Sequential([keras_layer]) - keras_model.build([None, 224, 224, 3]) - input_tensor = tf.ones((4, 224, 224, 3)) - output_tensor = keras_model(input_tensor) - print("output_tensor.shape:", output_tensor.shape) - + """ + Download the MobileNetV4-Conv-Large-fp32 from https://github.com/mlcommons/mobile_open/releases + Tested with tensorflow==2.15.0 and coremltools==7.1 + """ + saved_model_dir = '../dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-saved-model' + export_fpath = '../dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel' + print("Converting model...") model = ct.convert( - keras_model, + saved_model_dir, + source="tensorflow", convert_to="neuralnetwork", - inputs=[ct.TensorType(shape=(1, 224, 224, 3))], + inputs=[ct.TensorType(shape=(1, 384, 384, 3))], ) - model.short_description = "MobilenetEdgeTPUv2 from https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1" + model.short_description = "MobileNetV4-Conv-Large-fp32 from https://github.com/mlcommons/mobile_open" spec = model.get_spec() for n in (1, 1001): spec.description.output[0].type.multiArrayType.shape.append(n) - ct.utils.rename_feature(spec, "keras_layer_input", "images") + ct.utils.rename_feature(spec, "inputs", "images") ct.utils.rename_feature(spec, "Identity", "Softmax") print(spec.description) - export_fpath = '../dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel' ct.models.MLModel(spec).save(export_fpath) + print("Done! Core ML model exported to:", export_fpath) if __name__ == "__main__": diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 1022ac06f..6f07fee5f 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -31,43 +31,43 @@ benchmark_setting { } benchmark_setting { - benchmark_id: "image_classification_offline" + benchmark_id: "image_classification_v2" framework: "TFLite" delegate_choice: { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_uint8.tflite" - model_checksum: "008dfcb1c1962fedbeef1b998d4c84f2" - batch_size: 64 + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/main/v1_1/tflite/mobilenet_edgetpu_224_1.0_float.tflite" - model_checksum: "66bb4eba50987221608f8487ed405794" - batch_size: 2 + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "NNAPI" } benchmark_setting { - benchmark_id: "image_classification_v2" + benchmark_id: "image_classification_offline" framework: "TFLite" delegate_choice: { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_uint8.tflite" + model_checksum: "008dfcb1c1962fedbeef1b998d4c84f2" + batch_size: 64 } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_models/raw/main/v1_1/tflite/mobilenet_edgetpu_224_1.0_float.tflite" + model_checksum: "66bb4eba50987221608f8487ed405794" + batch_size: 2 } delegate_selected: "NNAPI" } @@ -79,16 +79,16 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" batch_size: 16 + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 2 } delegate_selected: "NNAPI" @@ -112,7 +112,6 @@ benchmark_setting { model_checksum: "566ceb72a4c7c8926fe4ac8eededb5bf" } delegate_selected: "NNAPI" - } benchmark_setting { diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index 3bbe1ae17..7f0b3b8a6 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -38,15 +38,15 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "NNAPI" } @@ -80,16 +80,16 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" batch_size: 2 } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 2 } delegate_selected: "NNAPI" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt index 8669e7fb4..4b60e43c8 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 64 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 64 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt index 44b482330..2b77ca063 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 8 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 8 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt index f90255d22..6285f4eeb 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 32 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 32 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt index 42cd0e2d1..355aa3d42 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt @@ -38,15 +38,15 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "Core ML" } diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt index 2eae34450..6bfad7fd5 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt @@ -45,8 +45,8 @@ benchmark_setting { delegate_name: "CPU" accelerator_name: "cpu" accelerator_desc: "CPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "CPU" } @@ -59,8 +59,8 @@ benchmark_setting { accelerator_name: "cpu" accelerator_desc: "CPU" batch_size: 2 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "CPU" } diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt index 118e8609a..eddd47301 100644 --- a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt @@ -35,6 +35,33 @@ benchmark_setting { delegate_selected: "Neuron" } +benchmark_setting { + benchmark_id: "image_classification_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Neuron" + accelerator_name: "neuron-mdla" + accelerator_desc: "MediaTek NN accelerator via the Neuron Delegate" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + } + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "npu" + accelerator_desc: "NPU" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" + } + delegate_selected: "Neuron" +} + benchmark_setting { benchmark_id: "image_classification_offline" framework: "TFLite" @@ -63,6 +90,34 @@ benchmark_setting { delegate_selected: "Neuron" } +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Neuron" + accelerator_name: "neuron-mdla" + accelerator_desc: "MediaTek NN accelerator via the Neuron Delegate" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + batch_size: 256 + } + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "npu" + accelerator_desc: "NPU" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" + } + delegate_selected: "Neuron" +} + benchmark_setting { benchmark_id: "object_detection" framework: "TFLite"