Add model category attributes to model template (#2439)

Add model category attributes to model template * Add model category & status fields in model template * Add is_default_for_task attr to model template * Update model templates with category attrs * Add integration tests for model templates consistency * Fix license & doc string * Fix typo * Refactor test cases * Refactor common tests by generator --------- Signed-off-by: Songki Choi <[email protected]>
openvinotoolkit · Aug 22, 2023 · 1889b8c · 1889b8c
1 parent ded91a7
commit 1889b8c
Show file tree

Hide file tree

Showing 30 changed files with 286 additions and 103 deletions.
diff --git a/src/otx/algorithms/anomaly/configs/classification/padim/template.yaml b/src/otx/algorithms/anomaly/configs/classification/padim/template.yaml
@@ -29,3 +29,7 @@ training_targets:
 # Computational Complexity
 gigaflops: 3.9
 size: 168.4
+
+# Model spec
+model_category: SPEED
+is_default_for_task: true
diff --git a/src/otx/algorithms/anomaly/configs/classification/stfpm/template.yaml b/src/otx/algorithms/anomaly/configs/classification/stfpm/template.yaml
@@ -35,3 +35,6 @@ training_targets:
 # Computational Complexity
 gigaflops: 5.6
 size: 21.1
+
+# Model spec
+model_category: ACCURACY
diff --git a/src/otx/algorithms/anomaly/configs/detection/padim/template.yaml b/src/otx/algorithms/anomaly/configs/detection/padim/template.yaml
@@ -29,3 +29,7 @@ training_targets:
 # Computational Complexity
 gigaflops: 3.9
 size: 168.4
+
+# Model spec
+model_category: SPEED
+is_default_for_task: true
diff --git a/src/otx/algorithms/anomaly/configs/detection/stfpm/template.yaml b/src/otx/algorithms/anomaly/configs/detection/stfpm/template.yaml
@@ -35,3 +35,6 @@ training_targets:
 # Computational Complexity
 gigaflops: 5.6
 size: 21.1
+
+# Model spec
+model_category: ACCURACY
diff --git a/src/otx/algorithms/anomaly/configs/segmentation/padim/template.yaml b/src/otx/algorithms/anomaly/configs/segmentation/padim/template.yaml
@@ -29,3 +29,7 @@ training_targets:
 # Computational Complexity
 gigaflops: 3.9
 size: 168.4
+
+# Model spec
+model_category: SPEED
+is_default_for_task: true
diff --git a/src/otx/algorithms/anomaly/configs/segmentation/stfpm/template.yaml b/src/otx/algorithms/anomaly/configs/segmentation/stfpm/template.yaml
@@ -35,3 +35,6 @@ training_targets:
 # Computational Complexity
 gigaflops: 5.6
 size: 21.1
+
+# Model spec
+model_category: ACCURACY
diff --git a/src/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/template.yaml b/src/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/template.yaml
@@ -57,3 +57,7 @@ training_targets:
 # Stats.
 gigaflops: 0.81
 size: 4.09
+
+# Model spec
+model_category: BALANCE
+is_default_for_task: true
diff --git a/src/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/template.yaml b/src/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/template.yaml
@@ -57,3 +57,6 @@ training_targets:
 # Stats.
 gigaflops: 5.76
 size: 20.23
+
+# Model spec
+model_category: ACCURACY
diff --git a/src/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/template.yaml b/src/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/template.yaml
@@ -57,3 +57,6 @@ training_targets:
 # Stats.
 gigaflops: 0.44
 size: 4.29
+
+# Model spec
+model_category: SPEED
diff --git a/src/otx/algorithms/detection/configs/detection/cspdarknet_yolox/template.yaml b/src/otx/algorithms/detection/configs/detection/cspdarknet_yolox/template.yaml
@@ -57,8 +57,6 @@ training_targets:
 # Stats.
 gigaflops: 6.5
 size: 20.4
-# # Inference options. Defined by OpenVINO capabilities, not Algo Backend or Platform.
-# inference_targets:
-#   - CPU
-#   - GPU
-#   - VPU
+
+# Model spec
+model_category: SPEED
diff --git a/src/otx/algorithms/detection/configs/detection/mobilenetv2_atss/template.yaml b/src/otx/algorithms/detection/configs/detection/mobilenetv2_atss/template.yaml
@@ -57,8 +57,7 @@ training_targets:
 # Stats.
 gigaflops: 20.6
 size: 9.1
-# # Inference options. Defined by OpenVINO capabilities, not Algo Backend or Platform.
-# inference_targets:
-#   - CPU
-#   - GPU
-#   - VPU
+
+# Model spec
+model_category: ACCURACY
+is_default_for_task: true
diff --git a/src/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/template.yaml b/src/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/template.yaml
@@ -57,8 +57,6 @@ training_targets:
 # Stats.
 gigaflops: 9.4
 size: 7.6
-# # Inference options. Defined by OpenVINO capabilities, not Algo Backend or Platform.
-# inference_targets:
-#   - CPU
-#   - GPU
-#   - VPU
+
+# Model spec
+model_category: BALANCE
diff --git a/...algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/template.yaml b/...algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/template.yaml
@@ -60,3 +60,6 @@ training_targets:
 # Stats.
 gigaflops: 68.48
 size: 13.27
+
+# Model spec
+model_category: SPEED
diff --git a/src/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/template.yaml b/src/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/template.yaml
@@ -60,3 +60,7 @@ training_targets:
 # Stats.
 gigaflops: 533.8
 size: 177.9
+
+# Model spec
+model_category: ACCURACY
+is_default_for_task: true
diff --git a/...otx/algorithms/detection/configs/rotated_detection/efficientnetb2b_maskrcnn/template.yaml b/...otx/algorithms/detection/configs/rotated_detection/efficientnetb2b_maskrcnn/template.yaml
@@ -60,3 +60,6 @@ training_targets:
 # Stats.
 gigaflops: 68.48
 size: 13.27
+
+# Model spec
+model_category: SPEED
diff --git a/src/otx/algorithms/detection/configs/rotated_detection/resnet50_maskrcnn/template.yaml b/src/otx/algorithms/detection/configs/rotated_detection/resnet50_maskrcnn/template.yaml
@@ -60,3 +60,7 @@ training_targets:
 # Stats.
 gigaflops: 533.8
 size: 177.9
+
+# Model spec
+model_category: ACCURACY
+is_default_for_task: true
diff --git a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_18_mod2/template.yaml b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_18_mod2/template.yaml
@@ -58,3 +58,7 @@ training_targets:
 # Stats.
 gigaflops: 3.63
 size: 4.8
+
+# Model spec
+model_category: BALANCE
+is_default_for_task: true
diff --git a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_s_mod2/template.yaml b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_s_mod2/template.yaml
@@ -59,3 +59,6 @@ training_targets:
 # Stats.
 gigaflops: 1.82
 size: 3.5
+
+# Model spec
+model_category: SPEED
diff --git a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_x_mod3/template.yaml b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_x_mod3/template.yaml
@@ -59,3 +59,6 @@ training_targets:
 # Stats.
 gigaflops: 13.97
 size: 6.4
+
+# Model spec
+model_category: ACCURACY
diff --git a/src/otx/api/entities/model_template.py b/src/otx/api/entities/model_template.py
@@ -1,6 +1,6 @@
 """This file defines the ModelConfiguration, ModelEntity and Model classes."""
 
-# Copyright (C) 2021-2022 Intel Corporation
+# Copyright (C) 2021-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
 import copy
@@ -459,6 +459,30 @@ class EntryPoints:
     nncf: Optional[str] = None
 
 
+class ModelCategory(Enum):
+    """Represents model category regarding accuracy & speed trade-off."""
+
+    SPEED = auto()
+    BALANCE = auto()
+    ACCURACY = auto()
+    OTHER = auto()
+
+    def __str__(self) -> str:
+        """Returns the name of the model category."""
+        return str(self.name)
+
+
+class ModelStatus(Enum):
+    """Represents model status regarding deprecation process."""
+
+    ACTIVE = auto()
+    DEPRECATED = auto()
+
+    def __str__(self) -> str:
+        """Returns the name of the model status."""
+        return str(self.name)
+
+
 # pylint: disable=too-many-instance-attributes
 @dataclass
 class ModelTemplate:
@@ -499,6 +523,9 @@ class ModelTemplate:
         priority. mobilenet is less important, and has a higher value. Default is zero (the highest priority).
     gigaflops (float): how many billions of operations are required to do inference on a single data item.
     size (float): how much disk space the model will approximately take.
+    model_category (ModelCategory): Represents model category regarding accuracy & speed trade-off. Default to OTHER.
+    model_status (ModelStatus): Represents model status regarding deprecation process. Default to ACTIVE.
+    is_default_for_task (bool): Whether this model is a default recommendation for the task
     """
 
     model_template_id: str
@@ -528,6 +555,9 @@ class ModelTemplate:
     gigaflops: float = 0
     size: float = 0
     hpo: Optional[Dict] = None
+    model_category: ModelCategory = ModelCategory.OTHER
+    model_status: ModelStatus = ModelStatus.ACTIVE
+    is_default_for_task: bool = False
 
     def __post_init__(self):
         """Do sanitation checks before loading the hyper-parameters."""

diff --git a/tests/integration/cli/anomaly/test_anomaly_classification.py b/tests/integration/cli/anomaly/test_anomaly_classification.py
@@ -1,23 +1,13 @@
 """Tests for anomaly classification with OTX CLI"""
-
-# Copyright (C) 2021 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
+# Copyright (C) 2021-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 #
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions
-# and limitations under the License.
 
 import os
 
 import pytest
 
+from otx.api.entities.model_template import parse_model_template
 from otx.cli.registry import Registry
 from tests.test_suite.e2e_test_system import e2e_pytest_component
 from tests.test_suite.run_test_command import (
@@ -28,6 +18,7 @@
     otx_eval_testing,
     otx_export_testing,
     otx_train_testing,
+    generate_model_template_testing,
 )
 
 args = {
@@ -44,6 +35,9 @@
 templates_ids = [template.model_template_id for template in templates]
 
 
+TestAnomalyClassificationModelTemplates = generate_model_template_testing(templates)
+
+
 class TestToolsAnomalyClassification:
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/tests/integration/cli/anomaly/test_anomaly_detection.py b/tests/integration/cli/anomaly/test_anomaly_detection.py
@@ -1,23 +1,13 @@
 """Tests for anomaly detection with OTX CLI."""
-
-# Copyright (C) 2021 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
+# Copyright (C) 2021-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 #
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions
-# and limitations under the License.
 
 import os
 
 import pytest
 
+from otx.api.entities.model_template import parse_model_template
 from otx.cli.registry import Registry
 from tests.test_suite.e2e_test_system import e2e_pytest_component
 from tests.test_suite.run_test_command import (
@@ -28,6 +18,7 @@
     otx_eval_testing,
     otx_export_testing,
     otx_train_testing,
+    generate_model_template_testing,
 )
 
 args = {
@@ -44,6 +35,9 @@
 templates_ids = [template.model_template_id for template in templates]
 
 
+TestAnomalyDetectionModelTemplates = generate_model_template_testing(templates)
+
+
 class TestToolsAnomalyDetection:
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/tests/integration/cli/anomaly/test_anomaly_segmentation.py b/tests/integration/cli/anomaly/test_anomaly_segmentation.py
@@ -1,23 +1,13 @@
 """Tests for anomaly segmentation with OTX CLI"""
-
-# Copyright (C) 2021 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
+# Copyright (C) 2021-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 #
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions
-# and limitations under the License.
 
 import os
 
 import pytest
 
+from otx.api.entities.model_template import parse_model_template
 from otx.cli.registry import Registry
 from tests.test_suite.e2e_test_system import e2e_pytest_component
 from tests.test_suite.run_test_command import (
@@ -28,6 +18,7 @@
     otx_eval_testing,
     otx_export_testing,
     otx_train_testing,
+    generate_model_template_testing,
 )
 
 args = {
@@ -44,6 +35,9 @@
 templates_ids = [template.model_template_id for template in templates]
 
 
+TestAnomalySegmentationModelTemplates = generate_model_template_testing(templates)
+
+
 class TestToolsAnomalySegmentation:
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/tests/integration/cli/classification/test_classification.py b/tests/integration/cli/classification/test_classification.py
@@ -1,5 +1,5 @@
 """Tests for Classification with OTX CLI"""
-# Copyright (C) 2022 Intel Corporation
+# Copyright (C) 2022-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
 
@@ -30,6 +30,7 @@
     otx_hpo_testing,
     otx_resume_testing,
     otx_train_testing,
+    generate_model_template_testing,
 )
 
 # Pre-train w/ 'label_0', 'label_1', 'label_2' classes
@@ -82,6 +83,9 @@
 templates_ids = [template.model_template_id for template in templates]
 
 
+TestClassificationModelTemplates = generate_model_template_testing(templates)
+
+
 class TestMultiClassClassificationCLI:
     @e2e_pytest_component
     @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids)

diff --git a/tests/integration/cli/detection/test_detection.py b/tests/integration/cli/detection/test_detection.py
@@ -1,5 +1,5 @@
 """Tests for Class-Incremental Learning for object detection with OTX CLI"""
-# Copyright (C) 2022 Intel Corporation
+# Copyright (C) 2022-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
 import copy
@@ -28,6 +28,7 @@
     otx_hpo_testing,
     otx_resume_testing,
     otx_train_testing,
+    generate_model_template_testing,
 )
 
 args = {
@@ -81,6 +82,9 @@
 templates_ids_w_experimental = templates_ids + experimental_template_ids
 
 
+TestDetectionModelTemplates = generate_model_template_testing(templates)
+
+
 class TestDetectionCLI:
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates_w_experimental, ids=templates_ids_w_experimental)