diff --git a/src/otx/algorithms/anomaly/configs/classification/padim/template.yaml b/src/otx/algorithms/anomaly/configs/classification/padim/template.yaml index b63060a72d5..21c557530a9 100644 --- a/src/otx/algorithms/anomaly/configs/classification/padim/template.yaml +++ b/src/otx/algorithms/anomaly/configs/classification/padim/template.yaml @@ -29,3 +29,7 @@ training_targets: # Computational Complexity gigaflops: 3.9 size: 168.4 + +# Model spec +model_category: SPEED +is_default_for_task: true diff --git a/src/otx/algorithms/anomaly/configs/classification/stfpm/template.yaml b/src/otx/algorithms/anomaly/configs/classification/stfpm/template.yaml index ccdffab89b9..42ef6129a32 100644 --- a/src/otx/algorithms/anomaly/configs/classification/stfpm/template.yaml +++ b/src/otx/algorithms/anomaly/configs/classification/stfpm/template.yaml @@ -35,3 +35,6 @@ training_targets: # Computational Complexity gigaflops: 5.6 size: 21.1 + +# Model spec +model_category: ACCURACY diff --git a/src/otx/algorithms/anomaly/configs/detection/padim/template.yaml b/src/otx/algorithms/anomaly/configs/detection/padim/template.yaml index 20b0823f922..93bb00a0dfd 100644 --- a/src/otx/algorithms/anomaly/configs/detection/padim/template.yaml +++ b/src/otx/algorithms/anomaly/configs/detection/padim/template.yaml @@ -29,3 +29,7 @@ training_targets: # Computational Complexity gigaflops: 3.9 size: 168.4 + +# Model spec +model_category: SPEED +is_default_for_task: true diff --git a/src/otx/algorithms/anomaly/configs/detection/stfpm/template.yaml b/src/otx/algorithms/anomaly/configs/detection/stfpm/template.yaml index da6075e438f..5d4d49832c0 100644 --- a/src/otx/algorithms/anomaly/configs/detection/stfpm/template.yaml +++ b/src/otx/algorithms/anomaly/configs/detection/stfpm/template.yaml @@ -35,3 +35,6 @@ training_targets: # Computational Complexity gigaflops: 5.6 size: 21.1 + +# Model spec +model_category: ACCURACY diff --git a/src/otx/algorithms/anomaly/configs/segmentation/padim/template.yaml b/src/otx/algorithms/anomaly/configs/segmentation/padim/template.yaml index 65026654fae..8ac2fdb8a13 100644 --- a/src/otx/algorithms/anomaly/configs/segmentation/padim/template.yaml +++ b/src/otx/algorithms/anomaly/configs/segmentation/padim/template.yaml @@ -29,3 +29,7 @@ training_targets: # Computational Complexity gigaflops: 3.9 size: 168.4 + +# Model spec +model_category: SPEED +is_default_for_task: true diff --git a/src/otx/algorithms/anomaly/configs/segmentation/stfpm/template.yaml b/src/otx/algorithms/anomaly/configs/segmentation/stfpm/template.yaml index 1c72b4ddad7..fa17fc6f07e 100644 --- a/src/otx/algorithms/anomaly/configs/segmentation/stfpm/template.yaml +++ b/src/otx/algorithms/anomaly/configs/segmentation/stfpm/template.yaml @@ -35,3 +35,6 @@ training_targets: # Computational Complexity gigaflops: 5.6 size: 21.1 + +# Model spec +model_category: ACCURACY diff --git a/src/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/template.yaml b/src/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/template.yaml index 129ba98a535..1781b32e82e 100644 --- a/src/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/template.yaml +++ b/src/otx/algorithms/classification/configs/efficientnet_b0_cls_incr/template.yaml @@ -57,3 +57,7 @@ training_targets: # Stats. gigaflops: 0.81 size: 4.09 + +# Model spec +model_category: BALANCE +is_default_for_task: true diff --git a/src/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/template.yaml b/src/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/template.yaml index c31b29bf2d1..514815b1631 100644 --- a/src/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/template.yaml +++ b/src/otx/algorithms/classification/configs/efficientnet_v2_s_cls_incr/template.yaml @@ -57,3 +57,6 @@ training_targets: # Stats. gigaflops: 5.76 size: 20.23 + +# Model spec +model_category: ACCURACY diff --git a/src/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/template.yaml b/src/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/template.yaml index 4def2ca9b2d..60573d606c5 100644 --- a/src/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/template.yaml +++ b/src/otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/template.yaml @@ -57,3 +57,6 @@ training_targets: # Stats. gigaflops: 0.44 size: 4.29 + +# Model spec +model_category: SPEED diff --git a/src/otx/algorithms/detection/configs/detection/cspdarknet_yolox/template.yaml b/src/otx/algorithms/detection/configs/detection/cspdarknet_yolox/template.yaml index f7cf03fcced..7b37b8773fe 100644 --- a/src/otx/algorithms/detection/configs/detection/cspdarknet_yolox/template.yaml +++ b/src/otx/algorithms/detection/configs/detection/cspdarknet_yolox/template.yaml @@ -57,8 +57,6 @@ training_targets: # Stats. gigaflops: 6.5 size: 20.4 -# # Inference options. Defined by OpenVINO capabilities, not Algo Backend or Platform. -# inference_targets: -# - CPU -# - GPU -# - VPU + +# Model spec +model_category: SPEED diff --git a/src/otx/algorithms/detection/configs/detection/mobilenetv2_atss/template.yaml b/src/otx/algorithms/detection/configs/detection/mobilenetv2_atss/template.yaml index 04d1a2f6e60..c13a6dd69e7 100644 --- a/src/otx/algorithms/detection/configs/detection/mobilenetv2_atss/template.yaml +++ b/src/otx/algorithms/detection/configs/detection/mobilenetv2_atss/template.yaml @@ -57,8 +57,7 @@ training_targets: # Stats. gigaflops: 20.6 size: 9.1 -# # Inference options. Defined by OpenVINO capabilities, not Algo Backend or Platform. -# inference_targets: -# - CPU -# - GPU -# - VPU + +# Model spec +model_category: ACCURACY +is_default_for_task: true diff --git a/src/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/template.yaml b/src/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/template.yaml index fb5d0a954fd..d44ce0e283d 100644 --- a/src/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/template.yaml +++ b/src/otx/algorithms/detection/configs/detection/mobilenetv2_ssd/template.yaml @@ -57,8 +57,6 @@ training_targets: # Stats. gigaflops: 9.4 size: 7.6 -# # Inference options. Defined by OpenVINO capabilities, not Algo Backend or Platform. -# inference_targets: -# - CPU -# - GPU -# - VPU + +# Model spec +model_category: BALANCE diff --git a/src/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/template.yaml b/src/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/template.yaml index c1e40ce6beb..6c318d2ee54 100644 --- a/src/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/template.yaml +++ b/src/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/template.yaml @@ -60,3 +60,6 @@ training_targets: # Stats. gigaflops: 68.48 size: 13.27 + +# Model spec +model_category: SPEED diff --git a/src/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/template.yaml b/src/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/template.yaml index 355aa291bed..985dce2a69a 100644 --- a/src/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/template.yaml +++ b/src/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/template.yaml @@ -60,3 +60,7 @@ training_targets: # Stats. gigaflops: 533.8 size: 177.9 + +# Model spec +model_category: ACCURACY +is_default_for_task: true diff --git a/src/otx/algorithms/detection/configs/rotated_detection/efficientnetb2b_maskrcnn/template.yaml b/src/otx/algorithms/detection/configs/rotated_detection/efficientnetb2b_maskrcnn/template.yaml index 62707ac4028..34891f7e14e 100644 --- a/src/otx/algorithms/detection/configs/rotated_detection/efficientnetb2b_maskrcnn/template.yaml +++ b/src/otx/algorithms/detection/configs/rotated_detection/efficientnetb2b_maskrcnn/template.yaml @@ -60,3 +60,6 @@ training_targets: # Stats. gigaflops: 68.48 size: 13.27 + +# Model spec +model_category: SPEED diff --git a/src/otx/algorithms/detection/configs/rotated_detection/resnet50_maskrcnn/template.yaml b/src/otx/algorithms/detection/configs/rotated_detection/resnet50_maskrcnn/template.yaml index 69cb5104514..31e74540bda 100644 --- a/src/otx/algorithms/detection/configs/rotated_detection/resnet50_maskrcnn/template.yaml +++ b/src/otx/algorithms/detection/configs/rotated_detection/resnet50_maskrcnn/template.yaml @@ -60,3 +60,7 @@ training_targets: # Stats. gigaflops: 533.8 size: 177.9 + +# Model spec +model_category: ACCURACY +is_default_for_task: true diff --git a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_18_mod2/template.yaml b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_18_mod2/template.yaml index ca8bd75be0a..37d39dffb86 100644 --- a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_18_mod2/template.yaml +++ b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_18_mod2/template.yaml @@ -58,3 +58,7 @@ training_targets: # Stats. gigaflops: 3.63 size: 4.8 + +# Model spec +model_category: BALANCE +is_default_for_task: true diff --git a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_s_mod2/template.yaml b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_s_mod2/template.yaml index 6e3de18b4f9..ac2d7a8518f 100644 --- a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_s_mod2/template.yaml +++ b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_s_mod2/template.yaml @@ -59,3 +59,6 @@ training_targets: # Stats. gigaflops: 1.82 size: 3.5 + +# Model spec +model_category: SPEED diff --git a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_x_mod3/template.yaml b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_x_mod3/template.yaml index 0d6d2e8f27c..8e24a5837af 100644 --- a/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_x_mod3/template.yaml +++ b/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_x_mod3/template.yaml @@ -59,3 +59,6 @@ training_targets: # Stats. gigaflops: 13.97 size: 6.4 + +# Model spec +model_category: ACCURACY diff --git a/src/otx/api/entities/model_template.py b/src/otx/api/entities/model_template.py index 03426cc64d8..fe7b100c7fb 100644 --- a/src/otx/api/entities/model_template.py +++ b/src/otx/api/entities/model_template.py @@ -1,6 +1,6 @@ """This file defines the ModelConfiguration, ModelEntity and Model classes.""" -# Copyright (C) 2021-2022 Intel Corporation +# Copyright (C) 2021-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # import copy @@ -459,6 +459,30 @@ class EntryPoints: nncf: Optional[str] = None +class ModelCategory(Enum): + """Represents model category regarding accuracy & speed trade-off.""" + + SPEED = auto() + BALANCE = auto() + ACCURACY = auto() + OTHER = auto() + + def __str__(self) -> str: + """Returns the name of the model category.""" + return str(self.name) + + +class ModelStatus(Enum): + """Represents model status regarding deprecation process.""" + + ACTIVE = auto() + DEPRECATED = auto() + + def __str__(self) -> str: + """Returns the name of the model status.""" + return str(self.name) + + # pylint: disable=too-many-instance-attributes @dataclass class ModelTemplate: @@ -499,6 +523,9 @@ class ModelTemplate: priority. mobilenet is less important, and has a higher value. Default is zero (the highest priority). gigaflops (float): how many billions of operations are required to do inference on a single data item. size (float): how much disk space the model will approximately take. + model_category (ModelCategory): Represents model category regarding accuracy & speed trade-off. Default to OTHER. + model_status (ModelStatus): Represents model status regarding deprecation process. Default to ACTIVE. + is_default_for_task (bool): Whether this model is a default recommendation for the task """ model_template_id: str @@ -528,6 +555,9 @@ class ModelTemplate: gigaflops: float = 0 size: float = 0 hpo: Optional[Dict] = None + model_category: ModelCategory = ModelCategory.OTHER + model_status: ModelStatus = ModelStatus.ACTIVE + is_default_for_task: bool = False def __post_init__(self): """Do sanitation checks before loading the hyper-parameters.""" diff --git a/tests/integration/cli/anomaly/test_anomaly_classification.py b/tests/integration/cli/anomaly/test_anomaly_classification.py index 05bcd4f723a..c9173e08128 100644 --- a/tests/integration/cli/anomaly/test_anomaly_classification.py +++ b/tests/integration/cli/anomaly/test_anomaly_classification.py @@ -1,23 +1,13 @@ """Tests for anomaly classification with OTX CLI""" - -# Copyright (C) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright (C) 2021-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. import os import pytest +from otx.api.entities.model_template import parse_model_template from otx.cli.registry import Registry from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( @@ -28,6 +18,7 @@ otx_eval_testing, otx_export_testing, otx_train_testing, + generate_model_template_testing, ) args = { @@ -44,6 +35,9 @@ templates_ids = [template.model_template_id for template in templates] +TestAnomalyClassificationModelTemplates = generate_model_template_testing(templates) + + class TestToolsAnomalyClassification: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/integration/cli/anomaly/test_anomaly_detection.py b/tests/integration/cli/anomaly/test_anomaly_detection.py index 944f49aa318..0cb33a51fd8 100644 --- a/tests/integration/cli/anomaly/test_anomaly_detection.py +++ b/tests/integration/cli/anomaly/test_anomaly_detection.py @@ -1,23 +1,13 @@ """Tests for anomaly detection with OTX CLI.""" - -# Copyright (C) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright (C) 2021-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. import os import pytest +from otx.api.entities.model_template import parse_model_template from otx.cli.registry import Registry from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( @@ -28,6 +18,7 @@ otx_eval_testing, otx_export_testing, otx_train_testing, + generate_model_template_testing, ) args = { @@ -44,6 +35,9 @@ templates_ids = [template.model_template_id for template in templates] +TestAnomalyDetectionModelTemplates = generate_model_template_testing(templates) + + class TestToolsAnomalyDetection: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/integration/cli/anomaly/test_anomaly_segmentation.py b/tests/integration/cli/anomaly/test_anomaly_segmentation.py index 3918291e25e..17483504515 100644 --- a/tests/integration/cli/anomaly/test_anomaly_segmentation.py +++ b/tests/integration/cli/anomaly/test_anomaly_segmentation.py @@ -1,23 +1,13 @@ """Tests for anomaly segmentation with OTX CLI""" - -# Copyright (C) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright (C) 2021-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. import os import pytest +from otx.api.entities.model_template import parse_model_template from otx.cli.registry import Registry from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( @@ -28,6 +18,7 @@ otx_eval_testing, otx_export_testing, otx_train_testing, + generate_model_template_testing, ) args = { @@ -44,6 +35,9 @@ templates_ids = [template.model_template_id for template in templates] +TestAnomalySegmentationModelTemplates = generate_model_template_testing(templates) + + class TestToolsAnomalySegmentation: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/integration/cli/classification/test_classification.py b/tests/integration/cli/classification/test_classification.py index 45a98517284..5104af6b40b 100644 --- a/tests/integration/cli/classification/test_classification.py +++ b/tests/integration/cli/classification/test_classification.py @@ -1,5 +1,5 @@ """Tests for Classification with OTX CLI""" -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2022-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # @@ -30,6 +30,7 @@ otx_hpo_testing, otx_resume_testing, otx_train_testing, + generate_model_template_testing, ) # Pre-train w/ 'label_0', 'label_1', 'label_2' classes @@ -82,6 +83,9 @@ templates_ids = [template.model_template_id for template in templates] +TestClassificationModelTemplates = generate_model_template_testing(templates) + + class TestMultiClassClassificationCLI: @e2e_pytest_component @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) diff --git a/tests/integration/cli/detection/test_detection.py b/tests/integration/cli/detection/test_detection.py index e69810a9cf0..b9ba80011ba 100644 --- a/tests/integration/cli/detection/test_detection.py +++ b/tests/integration/cli/detection/test_detection.py @@ -1,5 +1,5 @@ """Tests for Class-Incremental Learning for object detection with OTX CLI""" -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2022-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # import copy @@ -28,6 +28,7 @@ otx_hpo_testing, otx_resume_testing, otx_train_testing, + generate_model_template_testing, ) args = { @@ -81,6 +82,9 @@ templates_ids_w_experimental = templates_ids + experimental_template_ids +TestDetectionModelTemplates = generate_model_template_testing(templates) + + class TestDetectionCLI: @e2e_pytest_component @pytest.mark.parametrize("template", templates_w_experimental, ids=templates_ids_w_experimental) diff --git a/tests/integration/cli/instance_segmentation/test_instance_segmentation.py b/tests/integration/cli/instance_segmentation/test_instance_segmentation.py index e5958fa2d04..c2ba3b9c3df 100644 --- a/tests/integration/cli/instance_segmentation/test_instance_segmentation.py +++ b/tests/integration/cli/instance_segmentation/test_instance_segmentation.py @@ -1,5 +1,5 @@ """Tests for Class-Incremental Learning for object detection with OTX CLI""" -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2022-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # import copy @@ -24,6 +24,7 @@ otx_hpo_testing, otx_resume_testing, otx_train_testing, + generate_model_template_testing, ) args = { @@ -67,6 +68,9 @@ templates_ids_inc_convnext.extend([template_experimental.model_template_id]) +TestInstanceSegmentationModelTemplates = generate_model_template_testing(templates) + + class TestInstanceSegmentationCLI: @e2e_pytest_component @pytest.mark.parametrize("template", templates_inc_convnext, ids=templates_ids_inc_convnext) diff --git a/tests/integration/cli/instance_segmentation/test_rotated_detection.py b/tests/integration/cli/instance_segmentation/test_rotated_detection.py new file mode 100644 index 00000000000..6e89e7f67a8 --- /dev/null +++ b/tests/integration/cli/instance_segmentation/test_rotated_detection.py @@ -0,0 +1,25 @@ +"""Tests for rotated object detection with OTX CLI""" +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +import copy +import os + +import pytest +import torch + +from otx.api.entities.model_template import parse_model_template +from otx.cli.registry import Registry +from tests.test_suite.e2e_test_system import e2e_pytest_component +from tests.test_suite.run_test_command import generate_model_template_testing + + +otx_dir = os.getcwd() + +MULTI_GPU_UNAVAILABLE = torch.cuda.device_count() <= 1 + +templates = Registry("src/otx/algorithms/detection").filter(task_type="ROTATED_DETECTION").templates +templates_ids = [template.model_template_id for template in templates] + + +TestRotatedDetectionModelTemplates = generate_model_template_testing(templates) diff --git a/tests/integration/cli/semantic_segmentation/test_segmentation.py b/tests/integration/cli/semantic_segmentation/test_segmentation.py index d558c39a15e..81884f948d8 100644 --- a/tests/integration/cli/semantic_segmentation/test_segmentation.py +++ b/tests/integration/cli/semantic_segmentation/test_segmentation.py @@ -1,5 +1,5 @@ """Tests for Semantic segmentation with OTX CLI""" -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2022-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # import copy @@ -9,6 +9,7 @@ import torch from otx.api.entities.model_template import parse_model_template +from otx.cli.registry import Registry from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( get_template_dir, @@ -21,6 +22,7 @@ otx_hpo_testing, otx_resume_testing, otx_train_testing, + generate_model_template_testing, ) args = { @@ -71,8 +73,11 @@ default_template = parse_model_template( os.path.join("src/otx/algorithms/segmentation/configs", "ocr_lite_hrnet_18_mod2", "template.yaml") ) -templates = [default_template] -templates_ids = [default_template.model_template_id] +default_templates = [default_template] +default_templates_ids = [default_template.model_template_id] + +templates = Registry("src/otx/algorithms/segmentation").filter(task_type="SEGMENTATION").templates +templates_ids = [template.model_template_id for template in templates] # add integration test for semi-sl with new SegNext model and prototype based approach # other tests will be updated accordingly after fully transfer to segnext templates @@ -83,22 +88,25 @@ templates_ids_inc_segnext = [segnext_experimental_template.model_template_id, default_template.model_template_id] +TestSemanticSegmentationModelTemplates = generate_model_template_testing(templates) + + class TestSegmentationCLI: @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_train_supcon(self, template, tmp_dir_path): args1 = copy.deepcopy(args) args1["train_params"].extend(["--learning_parameters.enable_supcon", "True"]) otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_train(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation" otx_train_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_resume(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation/test_resume" otx_resume_testing(template, tmp_dir_path, otx_dir, args) @@ -111,57 +119,57 @@ def test_otx_resume(self, template, tmp_dir_path): otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) @pytest.mark.parametrize("dump_features", [True, False]) def test_otx_export(self, template, tmp_dir_path, dump_features): tmp_dir_path = tmp_dir_path / "segmentation" otx_export_testing(template, tmp_dir_path, dump_features, check_ir_meta=True) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_export_fp16(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation" otx_export_testing(template, tmp_dir_path, half_precision=True) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_export_onnx(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation" otx_export_testing(template, tmp_dir_path, half_precision=False, is_onnx=True) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_eval(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation" otx_eval_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) @pytest.mark.parametrize("half_precision", [True, False]) def test_otx_eval_openvino(self, template, tmp_dir_path, half_precision): tmp_dir_path = tmp_dir_path / "segmentation" otx_eval_openvino_testing(template, tmp_dir_path, otx_dir, args, threshold=1.0, half_precision=half_precision) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_deploy_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation" otx_deploy_openvino_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_eval_deployment(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation" otx_eval_deployment_testing(template, tmp_dir_path, otx_dir, args, threshold=1.0) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_hpo(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation/test_hpo" otx_hpo_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_nncf_optimize(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation" if template.entrypoints.nncf is None: @@ -171,7 +179,7 @@ def test_nncf_optimize(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(MULTI_GPU_UNAVAILABLE, reason="The number of gpu is insufficient") - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_multi_gpu_train(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation/test_multi_gpu" args1 = copy.deepcopy(args) @@ -200,14 +208,14 @@ def test_otx_multi_gpu_train_semisl(self, template, tmp_dir_path): assert os.path.exists(f"{template_dir}/semisl") @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_train_selfsl(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation/test_selfsl" otx_train_testing(template, tmp_dir_path, otx_dir, args_selfsl) @e2e_pytest_component @pytest.mark.skipif(MULTI_GPU_UNAVAILABLE, reason="The number of gpu is insufficient") - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_multi_gpu_train_selfsl(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "segmentation/test_multi_gpu_selfsl" args_selfsl_multigpu = copy.deepcopy(args_selfsl) @@ -215,7 +223,7 @@ def test_otx_multi_gpu_train_selfsl(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args_selfsl_multigpu) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) @pytest.mark.parametrize("bs_adapt_type", ["Safe", "Full"]) def test_otx_train_auto_adapt_batch_size(self, template, tmp_dir_path, bs_adapt_type): adapting_bs_args = copy.deepcopy(args) @@ -224,7 +232,7 @@ def test_otx_train_auto_adapt_batch_size(self, template, tmp_dir_path, bs_adapt_ otx_train_testing(template, tmp_dir_path, otx_dir, adapting_bs_args) @e2e_pytest_component - @pytest.mark.parametrize("template", templates, ids=templates_ids) + @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids) def test_otx_train_auto_adapt_num_workers(self, template, tmp_dir_path): adapting_num_workers_args = copy.deepcopy(args) adapting_num_workers_args["train_params"].extend(["--learning_parameters.auto_num_workers", "True"]) diff --git a/tests/test_suite/run_test_command.py b/tests/test_suite/run_test_command.py index f2f3a1baa2b..8554366e587 100644 --- a/tests/test_suite/run_test_command.py +++ b/tests/test_suite/run_test_command.py @@ -1,16 +1,7 @@ -# Copyright (C) 2021 Intel Corporation +"""Common test case and helpersi for OTX""" +# Copyright (C) 2021-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. import asyncio import json @@ -25,9 +16,11 @@ import pytest import yaml +from otx.api.entities.model_template import ModelCategory, ModelStatus from otx.cli.tools.find import SUPPORTED_BACKBONE_BACKENDS as find_supported_backends from otx.cli.tools.find import SUPPORTED_TASKS as find_supported_tasks from otx.cli.utils.nncf import get_number_of_fakequantizers_in_xml +from tests.test_suite.e2e_test_system import e2e_pytest_component def get_template_rel_dir(template): @@ -1083,3 +1076,38 @@ def otx_train_auto_config(root, otx_dir: str, args: Dict[str, str], use_output: command_line.extend(["--workspace", f"{work_dir}"]) command_line.extend(args["train_params"]) check_run(command_line) + + +def generate_model_template_testing(templates): + class _TestModelTemplates: + @e2e_pytest_component + def test_model_category(self): + stat = { + ModelCategory.SPEED: 0, + ModelCategory.BALANCE: 0, + ModelCategory.ACCURACY: 0, + ModelCategory.OTHER: 0, + } + for template in templates: + stat[template.model_category] += 1 + assert stat[ModelCategory.SPEED] == 1 + assert stat[ModelCategory.BALANCE] <= 1 + assert stat[ModelCategory.ACCURACY] == 1 + + @e2e_pytest_component + def test_model_status(self): + for template in templates: + if template.model_status == ModelStatus.DEPRECATED: + assert template.model_category == ModelCategory.OTHER + + @e2e_pytest_component + def test_default_for_task(self): + num_default_model = 0 + for template in templates: + if template.is_default_for_task: + num_default_model += 1 + assert template.model_category != ModelCategory.OTHER + assert template.model_status == ModelStatus.ACTIVE + assert num_default_model == 1 + + return _TestModelTemplates diff --git a/tests/unit/api/entities/test_model_template.py b/tests/unit/api/entities/test_model_template.py index 820d2d64a44..d00b4f5dc19 100644 --- a/tests/unit/api/entities/test_model_template.py +++ b/tests/unit/api/entities/test_model_template.py @@ -1,17 +1,7 @@ -# Copyright (C) 2020-2021 Intel Corporation +"""Tests for model template entity""" +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. - import copy import itertools from os import remove @@ -31,6 +21,8 @@ HyperParameterData, InstantiationType, ModelOptimizationMethod, + ModelCategory, + ModelStatus, ModelTemplate, NullModelTemplate, TargetDevice, @@ -97,6 +89,9 @@ def check_model_attributes(model: ModelTemplate, expected_values: dict): assert model.task_type_sort_priority == expected_values.get("task_type_sort_priority", -1) assert model.gigaflops == expected_values.get("gigaflops", 0) assert model.size == expected_values.get("size", 0) + assert model.model_category == expected_values.get("model_category", ModelCategory.OTHER) + assert model.model_status == expected_values.get("model_status", ModelStatus.ACTIVE) + assert model.is_default_for_task == expected_values.get("is_default_for_task", False) @pytest.mark.components(OtxSdkComponent.OTX_API) @@ -726,6 +721,56 @@ def test_entrypoints(self): ) +@pytest.mark.components(OtxSdkComponent.OTX_API) +class TestModelCategory: + @pytest.mark.priority_medium + @pytest.mark.unit + @pytest.mark.reqids(Requirements.REQ_1) + def test_model_category(self): + """ + Description: + Check ModelCategory Enum class elements + Expected results: + Test passes if ModelCategory Enum class length, attributes and methods return expected values + Steps + 1. Check ModelCategory length + 2. Check ModelCategory elements value attribute + 3. Check ModelCategory str method + """ + assert len(ModelCategory) == 4 + assert ModelCategory.SPEED.value == 1 + assert ModelCategory.BALANCE.value == 2 + assert ModelCategory.ACCURACY.value == 3 + assert ModelCategory.OTHER.value == 4 + assert str(ModelCategory.SPEED) == "SPEED" + assert str(ModelCategory.BALANCE) == "BALANCE" + assert str(ModelCategory.ACCURACY) == "ACCURACY" + assert str(ModelCategory.OTHER) == "OTHER" + + +@pytest.mark.components(OtxSdkComponent.OTX_API) +class TestModelStatus: + @pytest.mark.priority_medium + @pytest.mark.unit + @pytest.mark.reqids(Requirements.REQ_1) + def test_model_category(self): + """ + Description: + Check ModelStatus Enum class elements + Expected results: + Test passes if ModelStatus Enum class length, attributes and methods return expected values + Steps + 1. Check ModelStatus length + 2. Check ModelStatus elements value attribute + 3. Check ModelStatus str method + """ + assert len(ModelStatus) == 2 + assert ModelStatus.ACTIVE.value == 1 + assert ModelStatus.DEPRECATED.value == 2 + assert str(ModelStatus.ACTIVE) == "ACTIVE" + assert str(ModelStatus.DEPRECATED) == "DEPRECATED" + + @pytest.mark.components(OtxSdkComponent.OTX_API) class TestModelTemplate: @staticmethod @@ -788,6 +833,9 @@ def optional_model_parameters(self): optional_parameters["task_type_sort_priority"] = 0 optional_parameters["gigaflops"] = 1 optional_parameters["size"] = 1024 + optional_parameters["model_category"] = ModelCategory.SPEED + optional_parameters["model_status"] = ModelStatus.ACTIVE + optional_parameters["is_default_for_task"] = False return optional_parameters @pytest.mark.priority_medium