diff --git a/.azure_pipelines/job_templates/olive-example-template.yaml b/.azure_pipelines/job_templates/olive-example-template.yaml index c121761a1..ca710d2ce 100644 --- a/.azure_pipelines/job_templates/olive-example-template.yaml +++ b/.azure_pipelines/job_templates/olive-example-template.yaml @@ -43,14 +43,17 @@ jobs: displayName: Set exampleRequirements condition: eq(variables['exampleRequirements'], '') + - script: | + python -m pip install pytest + python -m pip install azure-storage-blob tabulate + python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/$(exampleRequirements) + displayName: Install Test Dependencies + - task: AzureCLI@1 inputs: azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) scriptLocation: 'inlineScript' inlineScript: | - python -m pip install pytest - python -m pip install azure-storage-blob tabulate - python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/$(exampleRequirements) python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/test_$(exampleName).py displayName: Test Examples env: diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index cdabbe058..825936230 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -97,6 +97,12 @@ jobs: resnet_ptq_cpu: exampleFolder: resnet exampleName: resnet_ptq_cpu + resnet_ptq_cpu_aml: + exampleFolder: resnet + exampleName: resnet_ptq_cpu_aml + resnet_ptq_cpu_aml_dataset: + exampleFolder: resnet + exampleName: resnet_ptq_cpu_aml_dataset resnet_vitis_ai_ptq_cpu: exampleFolder: resnet exampleName: resnet_vitis_ai_ptq_cpu @@ -129,6 +135,12 @@ jobs: resnet_ptq_cpu: exampleFolder: resnet exampleName: resnet_ptq_cpu + resnet_ptq_cpu_aml: + exampleFolder: resnet + exampleName: resnet_ptq_cpu_aml + resnet_ptq_cpu_aml_dataset: + exampleFolder: resnet + exampleName: resnet_ptq_cpu_aml_dataset resnet_qat: exampleFolder: resnet exampleName: resnet_qat diff --git a/examples/test/test_resnet_ptq_cpu.py b/examples/test/test_resnet_ptq_cpu.py index 00f46c6fa..a39cfcd1a 100644 --- a/examples/test/test_resnet_ptq_cpu.py +++ b/examples/test/test_resnet_ptq_cpu.py @@ -30,8 +30,8 @@ def setup(): @pytest.mark.parametrize("search_algorithm", ["random"]) @pytest.mark.parametrize("execution_order", ["pass-by-pass"]) -@pytest.mark.parametrize("system", ["local_system", "aml_system"]) -@pytest.mark.parametrize("olive_json", ["resnet_ptq_cpu.json", "resnet_ptq_cpu_aml_dataset.json"]) +@pytest.mark.parametrize("system", ["local_system"]) +@pytest.mark.parametrize("olive_json", ["resnet_ptq_cpu.json"]) @pytest.mark.skipif( version.parse(OrtVersion) == version.parse("1.16.0"), reason="resnet is not supported in ORT 1.16.0 caused by https://github.com/microsoft/onnxruntime/issues/17627", diff --git a/examples/test/test_resnet_ptq_cpu_aml.py b/examples/test/test_resnet_ptq_cpu_aml.py new file mode 100644 index 000000000..9fbbdf1cb --- /dev/null +++ b/examples/test/test_resnet_ptq_cpu_aml.py @@ -0,0 +1,50 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +import os +from pathlib import Path + +import pytest +from onnxruntime import __version__ as OrtVersion +from packaging import version +from utils import check_output, patch_config, set_azure_identity_logging + +from olive.common.utils import retry_func, run_subprocess + + +@pytest.fixture(scope="module", autouse=True) +def setup(): + """Setups any state specific to the execution of the given module.""" + cur_dir = Path(__file__).resolve().parent.parent + example_dir = cur_dir / "resnet" + os.chdir(example_dir) + + # prepare model and data + # retry since it fails randomly + retry_func(run_subprocess, kwargs={"cmd": "python prepare_model_data.py", "check": True}) + set_azure_identity_logging() + + yield + os.chdir(cur_dir) + + +# TODO(myguo): consider split the test into two tests if the CredentialUnavailableError still happens in Windows. +@pytest.mark.parametrize( + ("olive_json", "search_algorithm", "execution_order", "system"), + [ + ("resnet_ptq_cpu.json", "random", "pass-by-pass", "aml_system"), + ], +) +@pytest.mark.skipif( + version.parse(OrtVersion) == version.parse("1.16.0"), + reason="resnet is not supported in ORT 1.16.0 caused by https://github.com/microsoft/onnxruntime/issues/17627", +) +def test_resnet(search_algorithm, execution_order, system, olive_json): + # TODO(jambayk): add gpu e2e test + from olive.workflows import run as olive_run + + olive_config = patch_config(olive_json, search_algorithm, execution_order, system) + + footprint = olive_run(olive_config) + check_output(footprint) diff --git a/examples/test/test_resnet_ptq_cpu_aml_dataset.py b/examples/test/test_resnet_ptq_cpu_aml_dataset.py new file mode 100644 index 000000000..a3810305f --- /dev/null +++ b/examples/test/test_resnet_ptq_cpu_aml_dataset.py @@ -0,0 +1,51 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +import os +from pathlib import Path + +import pytest +from onnxruntime import __version__ as OrtVersion +from packaging import version +from utils import check_output, patch_config, set_azure_identity_logging + +from olive.common.utils import retry_func, run_subprocess + + +@pytest.fixture(scope="module", autouse=True) +def setup(): + """Setups any state specific to the execution of the given module.""" + cur_dir = Path(__file__).resolve().parent.parent + example_dir = cur_dir / "resnet" + os.chdir(example_dir) + + # prepare model and data + # retry since it fails randomly + retry_func(run_subprocess, kwargs={"cmd": "python prepare_model_data.py", "check": True}) + set_azure_identity_logging() + + yield + os.chdir(cur_dir) + + +# TODO(myguo): consider split the test into two tests if the CredentialUnavailableError still happens in Windows. +@pytest.mark.parametrize( + ("olive_json", "search_algorithm", "execution_order", "system"), + [ + ("resnet_ptq_cpu_aml_dataset.json", False, None, "local_system"), + ("resnet_ptq_cpu_aml_dataset.json", False, None, "aml_system"), + ], +) +@pytest.mark.skipif( + version.parse(OrtVersion) == version.parse("1.16.0"), + reason="resnet is not supported in ORT 1.16.0 caused by https://github.com/microsoft/onnxruntime/issues/17627", +) +def test_resnet(search_algorithm, execution_order, system, olive_json): + # TODO(jambayk): add gpu e2e test + from olive.workflows import run as olive_run + + olive_config = patch_config(olive_json, search_algorithm, execution_order, system) + + footprint = olive_run(olive_config) + check_output(footprint) diff --git a/examples/test/utils.py b/examples/test/utils.py index 594e96ef1..943d90206 100644 --- a/examples/test/utils.py +++ b/examples/test/utils.py @@ -3,7 +3,9 @@ # Licensed under the MIT License. # -------------------------------------------------------------------------- import json +import logging import os +import sys # pylint: disable=broad-exception-raised @@ -79,7 +81,11 @@ def update_azureml_config(olive_config): "workspace_name": workspace_name, # pipeline agents have managed identities which take precedence over the Azure CLI credentials # so we need to exclude managed identity credentials - "default_auth_params": {"exclude_managed_identity_credential": True}, + "default_auth_params": { + "exclude_environment_credential": True, + "exclude_managed_identity_credential": True, + "exclude_shared_token_cache_credential": True, + }, } @@ -149,3 +155,10 @@ def download_azure_blob(container, blob, download_path): with open(download_path, "wb") as my_blob: blob_data = blob.download_blob() blob_data.readinto(my_blob) + + +def set_azure_identity_logging(): + identity_logger = logging.getLogger("azure.identity") + identity_logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler(stream=sys.stdout) + identity_logger.addHandler(handler) diff --git a/olive/azureml/azureml_client.py b/olive/azureml/azureml_client.py index 4956838e7..188da2633 100644 --- a/olive/azureml/azureml_client.py +++ b/olive/azureml/azureml_client.py @@ -80,9 +80,7 @@ def create_client(self): """Create an MLClient instance.""" from azure.ai.ml import MLClient - # set logger level to error to avoid too many logs from azure sdk - logging.getLogger("azure.ai.ml").setLevel(logging.ERROR) - logging.getLogger("azure.identity").setLevel(logging.ERROR) + set_azure_logging_if_noset() if self.aml_config_path is None: if self.subscription_id is None: @@ -109,10 +107,7 @@ def create_registry_client(self, registry_name: str): """Create an MLClient instance.""" from azure.ai.ml import MLClient - # set logger level to error to avoid too many logs from azure sdk - logging.getLogger("azure.ai.ml").setLevel(logging.ERROR) - logging.getLogger("azure.identity").setLevel(logging.ERROR) - + set_azure_logging_if_noset() return MLClient(credential=self._get_credentials(), registry_name=registry_name) def _get_credentials(self): @@ -131,10 +126,21 @@ def _get_credentials(self): credential = DefaultAzureCredential(**default_auth_params) # Check if given credential can get token successfully. credential.get_token("https://management.azure.com/.default") - logger.debug("Using DefaultAzureCredential") except Exception: logger.warning("Using InteractiveBrowserCredential since of default credential errors", exc_info=True) # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work credential = InteractiveBrowserCredential() return credential + + +def set_azure_logging_if_noset(): + # set logger level to error to avoid too many logs from azure sdk + azure_ml_logger = logging.getLogger("azure.ai.ml") + # only set the level if it is not set, to avoid changing the level set by the user + if not azure_ml_logger.level: + azure_ml_logger.setLevel(logging.ERROR) + azure_identity_logger = logging.getLogger("azure.identity") + # only set the level if it is not set, to avoid changing the level set by the user + if not azure_identity_logger.level: + azure_identity_logger.setLevel(logging.WARNING)