For Mobilenet V4 large

.github/workflows/android-build-test.yml

@@ -86,23 +86,23 @@ jobs:
           python3 -m pip install awscli
       - name: Download Samsung libraries
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }}
         run: |
-          aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \
+          aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \
           cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/samsung_libs_v3.1_20231116.zip /tmp/ && \
           unzip /tmp/samsung_libs_v3.1_20231116.zip -d /tmp/samsung_libs_v3.1_20231116 && \
           rm /tmp/samsung_libs_v3.1_20231116.zip && \
           mkdir -p mobile_back_samsung/samsung/lib/internal && \
           mv /tmp/samsung_libs_v3.1_20231116/* mobile_back_samsung/samsung/lib/internal/
       - name: Download QTI libraries
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }}
         run: |
-          aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \
+          aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \
           cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/qaisw-2.13.4.230831213719_60417-linux.zip /tmp/ && \
-          unzip -P ${AWS_SECRET_ACCESS_KEY} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \
+          unzip -P ${{ secrets.READONLY_AWS_KEY }} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \
           rm /tmp/qaisw-2.13.4.230831213719_60417-linux.zip && \
           mv /tmp/qaisw-2.13.4.230831213719_60417-linux/qaisw-2.13.4.230831213719_60417 mobile_back_qti/
       - name: Cache bazel

flutter/assets/tasks.pbtxt

@@ -246,9 +246,9 @@ task {
   model {
     id: "MobileNetEdgeTPUv2-L"
     name: "MobileNetEdgeTPUv2-L"
-    offset: 1
-    image_width: 224
-    image_height: 224
+    offset: 0
+    image_width: 384
+    image_height: 384
     num_classes: 1000
   }
 }
@@ -283,9 +283,9 @@ task {
   model {
     id: "MobileNetEdgeTPUv2-L"
     name: "MobileNetEdgeTPUv2-L"
-    offset: 1
-    image_width: 224
-    image_height: 224
+    offset: 0
+    image_width: 384
+    image_height: 384
     num_classes: 1000
   }
 }

flutter/integration_test/expected_accuracy.dart

@@ -28,14 +28,14 @@ const Map<String, Interval> _imageClassification = {
 };
 
 const Map<String, Interval> _imageClassificationV2 = {
-  'cpu': Interval(min: 0.69, max: 0.71),
-  'npu': Interval(min: 0.69, max: 0.71),
-  'tpu': Interval(min: 0.69, max: 0.71),
-  'ane': Interval(min: 0.69, max: 0.71),
-  'cpu&gpu&ane': Interval(min: 0.69, max: 0.71),
-  'snpe_dsp': Interval(min: 0.69, max: 0.71),
-  'psnpe_dsp': Interval(min: 0.69, max: 0.71),
-  'neuron-mdla': Interval(min: 0.69, max: 0.71),
+  'cpu': Interval(min: 0.88, max: 0.91),
+  'npu': Interval(min: 0.88, max: 0.91),
+  'tpu': Interval(min: 0.88, max: 0.91),
+  'ane': Interval(min: 0.79, max: 0.91),
+  'cpu&gpu&ane': Interval(min: 0.79, max: 0.91),
+  'snpe_dsp': Interval(min: 0.88, max: 0.91),
+  'psnpe_dsp': Interval(min: 0.88, max: 0.91),
+  'neuron-mdla': Interval(min: 0.79, max: 0.81),
 };
 
 const Map<String, Interval> _objectDetection = {

flutter/integration_test/expected_throughput.dart

@@ -55,24 +55,24 @@ const Map<String, Map<String, Interval>> _imageClassification = {
 // TODO (anhappdev): update the expected value for _imageClassificationV2 after gathering some statistics
 const Map<String, Map<String, Interval>> _imageClassificationV2 = {
   _kTFLiteBackend: {
-    _kCloudBuildX23: Interval(min: 2, max: 11),
-    _kCloudBuildX28: Interval(min: 2, max: 12),
+    _kCloudBuildX23: Interval(min: 1, max: 9),
+    _kCloudBuildX28: Interval(min: 1, max: 9),
     _kRyzen5600: Interval(min: 1, max: 37),
-    _kPixel5: Interval(min: 60, max: 75),
-    _kIphoneOnGitHubAction: Interval(min: 1, max: 7),
+    _kPixel5: Interval(min: 20, max: 75),
+    _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7),
     _kIphoneOnMacbookM1: Interval(min: 10, max: 27),
   },
   _kCoreMLBackend: {
-    _kIphoneOnGitHubAction: Interval(min: 1, max: 7),
+    _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7),
   },
   _kPixelBackend: {
-    _kPixel6: Interval(min: 500, max: 600),
+    _kPixel6: Interval(min: 100, max: 600),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 1900, max: 2200),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 30, max: 50),
+    _kDN2103: Interval(min: 5, max: 50),
   },
 };
 
@@ -89,7 +89,7 @@ const Map<String, Map<String, Interval>> _objectDetection = {
     _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4),
   },
   _kPixelBackend: {
-    _kPixel6: Interval(min: 250, max: 450),
+    _kPixel6: Interval(min: 200, max: 450),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 800, max: 1400),
@@ -195,24 +195,24 @@ const Map<String, Map<String, Interval>> _imageClassificationOffline = {
 // TODO (anhappdev): update the expected value for _imageClassificationOfflineV2 after gathering some statistics
 const Map<String, Map<String, Interval>> _imageClassificationOfflineV2 = {
   _kTFLiteBackend: {
-    _kCloudBuildX23: Interval(min: 4, max: 14),
-    _kCloudBuildX28: Interval(min: 3, max: 14),
+    _kCloudBuildX23: Interval(min: 1, max: 9),
+    _kCloudBuildX28: Interval(min: 1, max: 9),
     _kRyzen5600: Interval(min: 20, max: 60),
-    _kPixel5: Interval(min: 60, max: 180),
-    _kIphoneOnGitHubAction: Interval(min: 1, max: 15),
+    _kPixel5: Interval(min: 20, max: 180),
+    _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15),
     _kIphoneOnMacbookM1: Interval(min: 10, max: 45),
   },
   _kCoreMLBackend: {
-    _kIphoneOnGitHubAction: Interval(min: 1, max: 15),
+    _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15),
   },
   _kPixelBackend: {
-    _kPixel6: Interval(min: 700, max: 1700),
+    _kPixel6: Interval(min: 100, max: 1700),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 2600, max: 3000),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 75, max: 110),
+    _kDN2103: Interval(min: 5, max: 90),
   },
 };
 

mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt

@@ -65,22 +65,22 @@ benchmark_setting {
     delegate_name: "CPU & GPU & ANE"
     accelerator_name: "cpu&gpu&ane"
     accelerator_desc: "All compute units"
-    model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel"
-    model_checksum: "875730de6fa07333051ca0f283fb23fd"
+    model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel"
+    model_checksum: "fbfba70596f74010852f22fa04721202"
   }
   delegate_choice: {
     delegate_name: "CPU & GPU"
     accelerator_name: "cpu&gpu"
     accelerator_desc: "CPU and GPU"
-    model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel"
-    model_checksum: "875730de6fa07333051ca0f283fb23fd"
+    model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel"
+    model_checksum: "fbfba70596f74010852f22fa04721202"
   }
   delegate_choice: {
     delegate_name: "CPU & ANE"
     accelerator_name: "cpu&ane"
     accelerator_desc: "CPU and Neural Engine"
-    model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel"
-    model_checksum: "875730de6fa07333051ca0f283fb23fd"
+    model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel"
+    model_checksum: "fbfba70596f74010852f22fa04721202"
   }
   delegate_selected: "CPU & GPU & ANE"
 }
@@ -93,24 +93,24 @@ benchmark_setting {
     accelerator_name: "cpu&gpu&ane"
     accelerator_desc: "All compute units"
     batch_size: 32
-    model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel"
-    model_checksum: "875730de6fa07333051ca0f283fb23fd"
+    model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel"
+    model_checksum: "fbfba70596f74010852f22fa04721202"
   }
   delegate_choice: {
     delegate_name: "CPU & GPU"
     accelerator_name: "cpu&gpu"
     accelerator_desc: "CPU and GPU"
     batch_size: 32
-    model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel"
-    model_checksum: "875730de6fa07333051ca0f283fb23fd"
+    model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel"
+    model_checksum: "fbfba70596f74010852f22fa04721202"
   }
   delegate_choice: {
     delegate_name: "CPU & ANE"
     accelerator_name: "cpu&ane"
     accelerator_desc: "CPU and Neural Engine"
     batch_size: 32
-    model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel"
-    model_checksum: "875730de6fa07333051ca0f283fb23fd"
+    model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel"
+    model_checksum: "fbfba70596f74010852f22fa04721202"
   }
   delegate_selected: "CPU & GPU & ANE"
 }

mobile_back_apple/dev-utils/Makefile

@@ -60,11 +60,13 @@ tflite-run-ic-v2:
 	bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification_v2 \
 		--mode=PerformanceOnly \
 		--output_dir="${REPO_ROOT_DIR}/output" \
-		--model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \
+		--model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-fp32.tflite" \
+		--image_width=384 \
+		--image_height=384 \
 		--lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \
 		--images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \
 		--groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \
-		--offset=1
+		--offset=0
 
 tflite-run-ic-offline-v2:
 	cd ${REPO_ROOT_DIR} && \
@@ -73,11 +75,13 @@ tflite-run-ic-offline-v2:
 		--scenario=Offline \
 		--batch_size=32 \
 		--output_dir="${REPO_ROOT_DIR}/output" \
-		--model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \
+		--model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-fp32.tflite" \
+		--image_width=384 \
+		--image_height=384 \
 		--lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \
 		--images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \
 		--groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \
-		--offset=1
+		--offset=0
 
 tflite-run-od:
 	cd ${REPO_ROOT_DIR} && \
@@ -155,11 +159,13 @@ coreml-run-ic-v2:
 	bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification \
 		--mode=PerformanceOnly \
 		--output_dir="${REPO_ROOT_DIR}/output" \
-		--model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \
+		--model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel" \
+		--image_width=384 \
+		--image_height=384 \
 		--lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \
 		--images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \
 		--groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \
-		--offset=1
+		--offset=0
 
 coreml-run-ic-offline-v2:
 	cd ${REPO_ROOT_DIR} && \
@@ -168,11 +174,13 @@ coreml-run-ic-offline-v2:
 		--scenario=Offline \
 		--batch_size=32 \
 		--output_dir="${REPO_ROOT_DIR}/output" \
-		--model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \
+		--model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel" \
+		--image_width=384 \
+		--image_height=384 \
 		--lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \
 		--images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \
 		--groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \
-		--offset=1
+		--offset=0
 
 coreml-run-od:
 	cd ${REPO_ROOT_DIR} && \

mobile_back_apple/models/icv2.py

@@ -16,36 +16,34 @@
 # limitations under the License.
 
 import coremltools as ct
-import tensorflow as tf
-import tensorflow_hub as hub
 
 
 def main():
-  print("Retrieve model from TFHub...")
-  keras_layer = hub.KerasLayer('https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1')
-  keras_model = tf.keras.Sequential([keras_layer])
-  keras_model.build([None, 224, 224, 3])
-  input_tensor = tf.ones((4, 224, 224, 3))
-  output_tensor = keras_model(input_tensor)
-  print("output_tensor.shape:", output_tensor.shape)
-
+  """
+  Download the MobileNetV4-Conv-Large-fp32 from https://github.com/mlcommons/mobile_open/releases
+  Tested with tensorflow==2.15.0 and coremltools==7.1
+  """
+  saved_model_dir = '../dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-saved-model'
+  export_fpath = '../dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel'
+  print("Converting model...")
   model = ct.convert(
-    keras_model,
+    saved_model_dir,
+    source="tensorflow",
     convert_to="neuralnetwork",
-    inputs=[ct.TensorType(shape=(1, 224, 224, 3))],
+    inputs=[ct.TensorType(shape=(1, 384, 384, 3))],
   )
-  model.short_description = "MobilenetEdgeTPUv2 from https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1"
+  model.short_description = "MobileNetV4-Conv-Large-fp32 from https://github.com/mlcommons/mobile_open"
 
   spec = model.get_spec()
   for n in (1, 1001):
     spec.description.output[0].type.multiArrayType.shape.append(n)
 
-  ct.utils.rename_feature(spec, "keras_layer_input", "images")
+  ct.utils.rename_feature(spec, "inputs", "images")
   ct.utils.rename_feature(spec, "Identity", "Softmax")
   print(spec.description)
 
-  export_fpath = '../dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel'
   ct.models.MLModel(spec).save(export_fpath)
+  print("Done! Core ML model exported to:", export_fpath)
 
 
 if __name__ == "__main__":