diff --git a/README.md b/README.md index 16a6e77dad2..6d3a8f45e76 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,29 @@ Please read the [Contribution guide](CONTRIBUTING.md) before starting work on a --- +## Telemetry data collection note + +The [OpenVINO™ telemetry library](https://github.com/openvinotoolkit/telemetry/) +is used to collect basic information about usage of this toolkit. + +The table below shows which event and data would be sent through telemetry data collection tool. +|Event|Description|Data Format| +|---|---|---| +|version|Installed version|\| +|success|Command that completed successfully|{'cmd': \, ['template': \], ['task_type': \]}| +|failure|Command that completed with an error|{'cmd': \, ['template': \], ['task_type': \]}| +|exception|Command that completed with unexpected exception|{'cmd': \, 'exception': \}| + +### How to control telemetry data collection + +To enable the collection of telemetry data, the consent file must exist and contain "1", otherwise telemetry will be disabled. The consent file can be created/modified by an OpenVINO installer or manually and used by other OpenVINO™ tools. + +The location and name of the consent file: + +`$HOME/intel/openvino_telemetry` + +--- + ## Known limitations Training, export, and evaluation scripts for TensorFlow- and most PyTorch-based models from the [misc](https://github.com/openvinotoolkit/training_extensions/tree/misc) branch are, currently, not production-ready. They serve exploratory purposes and are not validated. diff --git a/ote_cli/ote_cli/__init__.py b/ote_cli/ote_cli/__init__.py index 79931efa777..2ee421676f2 100644 --- a/ote_cli/ote_cli/__init__.py +++ b/ote_cli/ote_cli/__init__.py @@ -1,3 +1,12 @@ +"""OpenVINO Training Extensions.""" + # Copyright (C) 2021-2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # + +__version__ = "0.5.0" + + +def get_version(): + """retrun ote-cli version string""" + return __version__ diff --git a/ote_cli/ote_cli/tools/demo.py b/ote_cli/ote_cli/tools/demo.py index 0110c8da0b3..0efbf4fcda0 100644 --- a/ote_cli/ote_cli/tools/demo.py +++ b/ote_cli/ote_cli/tools/demo.py @@ -202,6 +202,8 @@ def main(): else: print(f"{frame_index=}, {elapsed_time=}, {len(predictions)=}") + return dict(retcode=0, template=template.name) + if __name__ == "__main__": main() diff --git a/ote_cli/ote_cli/tools/deploy.py b/ote_cli/ote_cli/tools/deploy.py index 2224f88e8ca..3d23b0e0bd2 100644 --- a/ote_cli/ote_cli/tools/deploy.py +++ b/ote_cli/ote_cli/tools/deploy.py @@ -89,6 +89,8 @@ def main(): with open(os.path.join(args.save_model_to, "openvino.zip"), "wb") as write_file: write_file.write(deployed_model.exportable_code) + return dict(retcode=0, template=template.name) + if __name__ == "__main__": main() diff --git a/ote_cli/ote_cli/tools/eval.py b/ote_cli/ote_cli/tools/eval.py index 0bb68a081d8..edc6e7f4c2f 100644 --- a/ote_cli/ote_cli/tools/eval.py +++ b/ote_cli/ote_cli/tools/eval.py @@ -176,6 +176,8 @@ def main(): write_file, ) + return dict(retcode=0, template=template.name) + if __name__ == "__main__": main() diff --git a/ote_cli/ote_cli/tools/export.py b/ote_cli/ote_cli/tools/export.py index fb4b8a665be..9e07c54a9f0 100644 --- a/ote_cli/ote_cli/tools/export.py +++ b/ote_cli/ote_cli/tools/export.py @@ -99,6 +99,8 @@ def main(): os.makedirs(args.save_model_to, exist_ok=True) save_model_data(exported_model, args.save_model_to) + return dict(retcode=0, template=template.name) + if __name__ == "__main__": main() diff --git a/ote_cli/ote_cli/tools/find.py b/ote_cli/ote_cli/tools/find.py index 0e60eb9221a..f8cd49f73d8 100644 --- a/ote_cli/ote_cli/tools/find.py +++ b/ote_cli/ote_cli/tools/find.py @@ -59,6 +59,8 @@ def main(): print(registry) + return dict(retcode=0, task_type=args.task_type) + if __name__ == "__main__": main() diff --git a/ote_cli/ote_cli/tools/optimize.py b/ote_cli/ote_cli/tools/optimize.py index 592e032a406..484472f0494 100644 --- a/ote_cli/ote_cli/tools/optimize.py +++ b/ote_cli/ote_cli/tools/optimize.py @@ -194,6 +194,8 @@ def main(): write_file, ) + return dict(retcode=0, template=template.name) + if __name__ == "__main__": main() diff --git a/ote_cli/ote_cli/tools/ote.py b/ote_cli/ote_cli/tools/ote.py index 7039e5b73a0..2bd6f069482 100644 --- a/ote_cli/ote_cli/tools/ote.py +++ b/ote_cli/ote_cli/tools/ote.py @@ -19,6 +19,8 @@ import argparse import sys +from ote_cli.utils import telemetry + from .demo import main as ote_demo from .deploy import main as ote_deploy from .eval import main as ote_eval @@ -64,7 +66,24 @@ def main(): name = parse_args().operation sys.argv[0] = f"ote {name}" del sys.argv[1] - globals()[f"ote_{name}"]() + + tm_session = telemetry.init_telemetry_session() + results = {} + try: + results = globals()[f"ote_{name}"]() + if results is None: + results = dict(retcode=0) + except Exception as error: + results["retcode"] = -1 + results["exception"] = repr(error) + telemetry.send_cmd_results(tm_session, name, results) + raise + else: + telemetry.send_cmd_results(tm_session, name, results) + finally: + telemetry.close_telemetry_session(tm_session) + + return results.get("retcode", 0) if __name__ == "__main__": diff --git a/ote_cli/ote_cli/tools/train.py b/ote_cli/ote_cli/tools/train.py index ceb5678be24..a29ed858cd2 100644 --- a/ote_cli/ote_cli/tools/train.py +++ b/ote_cli/ote_cli/tools/train.py @@ -203,6 +203,8 @@ def main(): assert resultset.performance is not None print(resultset.performance) + return dict(retcode=0, template=template.name) + if __name__ == "__main__": main() diff --git a/ote_cli/ote_cli/utils/telemetry.py b/ote_cli/ote_cli/utils/telemetry.py new file mode 100644 index 00000000000..2bccc475f14 --- /dev/null +++ b/ote_cli/ote_cli/utils/telemetry.py @@ -0,0 +1,92 @@ +# nosec + +# Copyright (C) 2021-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +""" +Utilities for OpenVINO telemetry +""" + +# pylint: disable=broad-except, ungrouped-imports +import json + +from ote_cli import get_version + +try: + import openvino_telemetry as tm +except ImportError: + from ote_cli.utils import telemetry_stub as tm + + +__TM_CATEGORY_OTX = "otx" +__TM_MEASUREMENT_ID = "UA-17808594-29" +# __TM_MEASUREMENT_ID_FOR_TESTING = "UA-254359572-1" + +__TM_ACTION_VERSION = "version" +__TM_ACTION_CMD_SUCCESS = "success" +__TM_ACTION_CMD_FAILURE = "failure" +__TM_ACTION_CMD_EXCEPTION = "exception" +__TM_ACTION_ERROR = "error" + + +def init_telemetry_session(): + """init session""" + telemetry = tm.Telemetry( + app_name=__TM_CATEGORY_OTX, app_version=get_version(), tid=__TM_MEASUREMENT_ID + ) + telemetry.start_session(__TM_CATEGORY_OTX) + send_version(telemetry) + + return telemetry + + +def close_telemetry_session(telemetry): + """close session""" + telemetry.end_session(__TM_CATEGORY_OTX) + telemetry.force_shutdown(1.0) + + +def send_version(telemetry): + """send application version""" + __send_event(telemetry, "version", str(get_version())) + + +def send_cmd_results(telemetry, cmd, results): + """send cli telemetry data""" + action = __TM_ACTION_ERROR + retcode = results.pop("retcode", None) + if retcode >= 0: + action = __TM_ACTION_CMD_FAILURE + if retcode == 0: + action = __TM_ACTION_CMD_SUCCESS + label = dict(cmd=cmd, **results) + elif retcode < 0: + action = __TM_ACTION_CMD_EXCEPTION + label = dict(cmd=cmd, **results) + + __send_event(telemetry, action, label) + + +def __send_event(telemetry, action, label, **kwargs): + """wrapper of the openvino-telemetry.send_event()""" + if not isinstance(action, str): + raise TypeError(f"action should string type but {type(action)}") + if not isinstance(label, dict) and not isinstance(label, str): + raise TypeError(f"label should 'dict' or 'str' type but {type(label)}") + + try: + telemetry.send_event(__TM_CATEGORY_OTX, action, json.dumps(label), **kwargs) + except Exception as error: + print(f"An error while calling otm.send_event(): \n{repr(error)}") + + +def __send_error(telemetry, err_msg, **kwargs): + """wrapper of the openvino-telemetry.send_error()""" + if not isinstance(err_msg, str): + raise TypeError(f"err_msg should string type but {type(err_msg)}") + + try: + telemetry.send_error(__TM_CATEGORY_OTX, err_msg, **kwargs) + except Exception as error: + print(f"An error while calling otm.send_error(): \n{repr(error)}") diff --git a/ote_cli/ote_cli/utils/telemetry_stub.py b/ote_cli/ote_cli/utils/telemetry_stub.py new file mode 100644 index 00000000000..2b07c6c2689 --- /dev/null +++ b/ote_cli/ote_cli/utils/telemetry_stub.py @@ -0,0 +1,31 @@ +# Copyright (C) 2021-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + + +class Telemetry(object): + """ + A stub for the Telemetry class, which is used when the Telemetry class + is not available. + """ + + def __init__(self, *arg, **kwargs): + pass + + def send_event(self, *arg, **kwargs): + pass + + def send_error(self, *arg, **kwargs): + pass + + def start_session(self, *arg, **kwargs): + pass + + def end_session(self, *arg, **kwargs): + pass + + def force_shutdown(self, *arg, **kwargs): + pass + + def send_stack_trace(self, *arg, **kwargs): + pass diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index 6097a016721..8649ef745c4 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -72,6 +72,7 @@ def collect_env_vars(work_dir): def check_run(cmd, **kwargs): result = subprocess.run(cmd, stderr=subprocess.PIPE, **kwargs) + print(f"***** result.returncode = {result.returncode} *****") assert result.returncode == 0, result.stderr.decode("utf=8") diff --git a/ote_cli/requirements.txt b/ote_cli/requirements.txt index 35145b49a17..7b01c5b5953 100644 --- a/ote_cli/requirements.txt +++ b/ote_cli/requirements.txt @@ -5,3 +5,4 @@ nbmake pytest pytest-ordering hpopt@git+https://github.com/openvinotoolkit/hyper_parameter_optimization@develop +openvino-telemetry>=2022.1.0 \ No newline at end of file diff --git a/ote_cli/setup.py b/ote_cli/setup.py index 482523f0690..98d831dfb5b 100644 --- a/ote_cli/setup.py +++ b/ote_cli/setup.py @@ -17,9 +17,46 @@ # and limitations under the License. import os +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path from setuptools import find_packages, setup + +def load_module(name: str = "ote_cli/__init__.py"): + """Load Python Module. + + Args: + name (str, optional): Name of the module to load. + Defaults to "ote_cli/__init__.py". + """ + location = str(Path(__file__).parent / name) + spec = spec_from_file_location(name=name, location=location) + module = module_from_spec(spec) # type: ignore + spec.loader.exec_module(module) # type: ignore + return module + + +def get_ote_cli_version() -> str: + """Get version from `ote_cli.__init__`. + + Version is stored in the main __init__ module in `ote_cli`. + The varible storing the version is `__version__`. This function + reads `__init__` file, checks `__version__ variable and return + the value assigned to it. + + Example: + >>> # Assume that __version__ = "0.2.6" + >>> get_ote_cli_version() + "0.2.6" + + Returns: + str: `ote_cli` version. + """ + ote_cli = load_module(name="ote_cli/__init__.py") + return ote_cli.__version__ + + with open( os.path.join(os.path.dirname(__file__), "requirements.txt"), encoding="UTF-8" ) as read_file: @@ -27,7 +64,7 @@ setup( name="ote_cli", - version="0.2", + version=get_ote_cli_version(), packages=find_packages(exclude=("tools",)), install_requires=requirements, entry_points={ diff --git a/tests/ote_cli/conftest.py b/tests/ote_cli/conftest.py index 67a5e30fc7f..feaffa585b6 100644 --- a/tests/ote_cli/conftest.py +++ b/tests/ote_cli/conftest.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions # and limitations under the License. +import os +import pytest + from ote_sdk.test_suite.pytest_insertions import * # noqa #pylint: disable=unused-import pytest_plugins = get_pytest_plugins_from_ote() @@ -21,3 +24,47 @@ def pytest_addoption(parser): ote_pytest_addoption_insertion(parser) + + +@pytest.fixture(autouse=True, scope="session") +def manage_tm_config_for_testing(): + # check file existance both 'isip' and 'openvino_telemetry' if not, create it. + # and backup contents if exist + cfg_dir = os.path.join(os.path.expanduser("~"), "intel") + isip_path = os.path.join(cfg_dir, "isip") + otm_path = os.path.join(cfg_dir, "openvino_telemetry") + isip_exist = os.path.exists(isip_path) + otm_exist = os.path.exists(otm_path) + + isip_backup = None + if not isip_exist: + with open(isip_path, "w") as f: + f.write("0") + else: + with open(isip_path, "r") as f: + isip_backup = f.read() + + otm_backup = None + if not otm_exist: + with open(otm_path, "w") as f: + f.write("0") + else: + with open(otm_path, "r") as f: + otm_backup = f.read() + + yield + + # restore or remove + if not isip_exist: + os.remove(isip_path) + else: + if isip_backup is not None: + with open(isip_path, "w") as f: + f.write(isip_backup) + + if not otm_exist: + os.remove(otm_path) + else: + if otm_backup is not None: + with open(otm_path, "w") as f: + f.write(otm_backup) diff --git a/tests/ote_cli/test_telemetry.py b/tests/ote_cli/test_telemetry.py new file mode 100644 index 00000000000..36c4238cdbd --- /dev/null +++ b/tests/ote_cli/test_telemetry.py @@ -0,0 +1,70 @@ +import sys +import subprocess +import pytest +import unittest +from unittest.mock import MagicMock, patch + +from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component + +from ote_cli.tools import ote + +from ote_cli.tools.ote import ( + ote_demo, + ote_deploy, + ote_eval, + ote_export, + ote_find, + ote_optimize, + ote_train +) + + +class TestTelemetry(unittest.TestCase): + + @e2e_pytest_component + @patch("ote_cli.tools.ote.ote_demo", return_value=None) + @patch("ote_cli.utils.telemetry.init_telemetry_session", return_value=None) + @patch("ote_cli.utils.telemetry.close_telemetry_session", return_value=None) + @patch("ote_cli.utils.telemetry.send_version", return_value=None) + @patch("ote_cli.utils.telemetry.send_cmd_results", return_value=None) + def test_tm_integration_exit_0(self, + mock_send_cmd, + mock_send_version, + mock_close_tm, + mock_init_tm, + mock_demo + ): + backup_argv = sys.argv + sys.argv = ["ote", "demo"] + ret = ote.main() + sys.argv = backup_argv + + self.assertEqual(ret, 0) + mock_demo.assert_called_once() + mock_init_tm.assert_called_once() + mock_close_tm.assert_called_once() + mock_send_cmd.assert_called_with(None, "demo", {"retcode": 0}) + + @e2e_pytest_component + @patch("ote_cli.tools.ote.ote_demo", side_effect=Exception()) + @patch("ote_cli.utils.telemetry.init_telemetry_session", return_value=None) + @patch("ote_cli.utils.telemetry.close_telemetry_session", return_value=None) + @patch("ote_cli.utils.telemetry.send_version", return_value=None) + @patch("ote_cli.utils.telemetry.send_cmd_results", return_value=None) + def test_tm_integration_exit_exception(self, + mock_send_cmd, + mock_send_version, + mock_close_tm, + mock_init_tm, + mock_demo, + ): + backup_argv = sys.argv + sys.argv = ["ote", "demo"] + with pytest.raises(Exception) as e: + ret = ote.main() + sys.argv = backup_argv + + self.assertEqual(e.type, Exception, f"{e}") + mock_init_tm.assert_called_once() + mock_close_tm.assert_called_once() + mock_send_cmd.assert_called_with(None, "demo", {"retcode": -1, "exception": repr(Exception())})