[develop] Integrate ov-telemetry

README.md

@@ -85,6 +85,29 @@ Please read the [Contribution guide](CONTRIBUTING.md) before starting work on a
 
 ---
 
+## Telemetry data collection note
+
+The [OpenVINO™ telemetry library](https://github.com/openvinotoolkit/telemetry/)
+is used to collect basic information about usage of this toolkit.
+
+The table below shows which event and data would be sent through telemetry data collection tool.
+|Event|Description|Data Format|
+|---|---|---|
+|version|Installed version|\<version-string\>|
+|success|Command that completed successfully|{'cmd': \<CLI-command-type\>, ['template': \<template-name\>], ['task_type': \<task-type-or-None\>]}|
+|failure|Command that completed with an error|{'cmd': \<CLI-command-type\>, ['template': \<template-name\>], ['task_type': \<task-type-or-None\>]}|
+|exception|Command that completed with unexpected exception|{'cmd': \<CLI-command-type\>, 'exception': \<exception-type-string\>}|
+
+### How to control telemetry data collection
+
+To enable the collection of telemetry data, the consent file must exist and contain "1", otherwise telemetry will be disabled. The consent file can be created/modified by an OpenVINO installer or manually and used by other OpenVINO™ tools.
+
+The location and name of the consent file:
+
+`$HOME/intel/openvino_telemetry`
+
+---
+
 ## Known limitations
 
 Training, export, and evaluation scripts for TensorFlow- and most PyTorch-based models from the [misc](https://github.com/openvinotoolkit/training_extensions/tree/misc) branch are, currently, not production-ready. They serve exploratory purposes and are not validated.

ote_cli/ote_cli/__init__.py

@@ -1,3 +1,12 @@
+"""OpenVINO Training Extensions."""
+
 # Copyright (C) 2021-2022 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
+
+__version__ = "0.5.0"
+
+
+def get_version():
+    """retrun ote-cli version string"""
+    return __version__

ote_cli/ote_cli/tools/demo.py

@@ -202,6 +202,8 @@ def main():
         else:
             print(f"{frame_index=}, {elapsed_time=}, {len(predictions)=}")
 
+    return dict(retcode=0, template=template.name)
+
 
 if __name__ == "__main__":
     main()

ote_cli/ote_cli/tools/deploy.py

@@ -89,6 +89,8 @@ def main():
     with open(os.path.join(args.save_model_to, "openvino.zip"), "wb") as write_file:
         write_file.write(deployed_model.exportable_code)
 
+    return dict(retcode=0, template=template.name)
+
 
 if __name__ == "__main__":
     main()

ote_cli/ote_cli/tools/eval.py

@@ -176,6 +176,8 @@ def main():
                 write_file,
             )
 
+    return dict(retcode=0, template=template.name)
+
 
 if __name__ == "__main__":
     main()

ote_cli/ote_cli/tools/export.py

@@ -99,6 +99,8 @@ def main():
     os.makedirs(args.save_model_to, exist_ok=True)
     save_model_data(exported_model, args.save_model_to)
 
+    return dict(retcode=0, template=template.name)
+
 
 if __name__ == "__main__":
     main()

ote_cli/ote_cli/tools/find.py

@@ -59,6 +59,8 @@ def main():
 
     print(registry)
 
+    return dict(retcode=0, task_type=args.task_type)
+
 
 if __name__ == "__main__":
     main()

ote_cli/ote_cli/tools/optimize.py

@@ -194,6 +194,8 @@ def main():
                 write_file,
             )
 
+    return dict(retcode=0, template=template.name)
+
 
 if __name__ == "__main__":
     main()

ote_cli/ote_cli/tools/ote.py

@@ -19,6 +19,8 @@
 import argparse
 import sys
 
+from ote_cli.utils import telemetry
+
 from .demo import main as ote_demo
 from .deploy import main as ote_deploy
 from .eval import main as ote_eval
@@ -64,7 +66,24 @@ def main():
     name = parse_args().operation
     sys.argv[0] = f"ote {name}"
     del sys.argv[1]
-    globals()[f"ote_{name}"]()
+
+    tm_session = telemetry.init_telemetry_session()
+    results = {}
+    try:
+        results = globals()[f"ote_{name}"]()
+        if results is None:
+            results = dict(retcode=0)
+    except Exception as error:
+        results["retcode"] = -1
+        results["exception"] = repr(error)
+        telemetry.send_cmd_results(tm_session, name, results)
+        raise
+    else:
+        telemetry.send_cmd_results(tm_session, name, results)
+    finally:
+        telemetry.close_telemetry_session(tm_session)
+
+    return results.get("retcode", 0)
 
 
 if __name__ == "__main__":

ote_cli/ote_cli/tools/train.py

@@ -203,6 +203,8 @@ def main():
     assert resultset.performance is not None
     print(resultset.performance)
 
+    return dict(retcode=0, template=template.name)
+
 
 if __name__ == "__main__":
     main()

ote_cli/ote_cli/utils/telemetry.py

@@ -0,0 +1,92 @@
+# nosec
+
+# Copyright (C) 2021-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+"""
+Utilities for OpenVINO telemetry
+"""
+
+# pylint: disable=broad-except, ungrouped-imports
+import json
+
+from ote_cli import get_version
+
+try:
+    import openvino_telemetry as tm
+except ImportError:
+    from ote_cli.utils import telemetry_stub as tm
+
+
+__TM_CATEGORY_OTX = "otx"
+__TM_MEASUREMENT_ID = "UA-17808594-29"
+# __TM_MEASUREMENT_ID_FOR_TESTING = "UA-254359572-1"
+
+__TM_ACTION_VERSION = "version"
+__TM_ACTION_CMD_SUCCESS = "success"
+__TM_ACTION_CMD_FAILURE = "failure"
+__TM_ACTION_CMD_EXCEPTION = "exception"
+__TM_ACTION_ERROR = "error"
+
+
+def init_telemetry_session():
+    """init session"""
+    telemetry = tm.Telemetry(
+        app_name=__TM_CATEGORY_OTX, app_version=get_version(), tid=__TM_MEASUREMENT_ID
+    )
+    telemetry.start_session(__TM_CATEGORY_OTX)
+    send_version(telemetry)
+
+    return telemetry
+
+
+def close_telemetry_session(telemetry):
+    """close session"""
+    telemetry.end_session(__TM_CATEGORY_OTX)
+    telemetry.force_shutdown(1.0)
+
+
+def send_version(telemetry):
+    """send application version"""
+    __send_event(telemetry, "version", str(get_version()))
+
+
+def send_cmd_results(telemetry, cmd, results):
+    """send cli telemetry data"""
+    action = __TM_ACTION_ERROR
+    retcode = results.pop("retcode", None)
+    if retcode >= 0:
+        action = __TM_ACTION_CMD_FAILURE
+        if retcode == 0:
+            action = __TM_ACTION_CMD_SUCCESS
+        label = dict(cmd=cmd, **results)
+    elif retcode < 0:
+        action = __TM_ACTION_CMD_EXCEPTION
+        label = dict(cmd=cmd, **results)
+
+    __send_event(telemetry, action, label)
+
+
+def __send_event(telemetry, action, label, **kwargs):
+    """wrapper of the openvino-telemetry.send_event()"""
+    if not isinstance(action, str):
+        raise TypeError(f"action should string type but {type(action)}")
+    if not isinstance(label, dict) and not isinstance(label, str):
+        raise TypeError(f"label should 'dict' or 'str' type but {type(label)}")
+
+    try:
+        telemetry.send_event(__TM_CATEGORY_OTX, action, json.dumps(label), **kwargs)
+    except Exception as error:
+        print(f"An error while calling otm.send_event(): \n{repr(error)}")
+
+
+def __send_error(telemetry, err_msg, **kwargs):
+    """wrapper of the openvino-telemetry.send_error()"""
+    if not isinstance(err_msg, str):
+        raise TypeError(f"err_msg should string type but {type(err_msg)}")
+
+    try:
+        telemetry.send_error(__TM_CATEGORY_OTX, err_msg, **kwargs)
+    except Exception as error:
+        print(f"An error while calling otm.send_error(): \n{repr(error)}")

ote_cli/ote_cli/utils/telemetry_stub.py

@@ -0,0 +1,31 @@
+# Copyright (C) 2021-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+
+class Telemetry(object):
+    """
+    A stub for the Telemetry class, which is used when the Telemetry class
+    is not available.
+    """
+
+    def __init__(self, *arg, **kwargs):
+        pass
+
+    def send_event(self, *arg, **kwargs):
+        pass
+
+    def send_error(self, *arg, **kwargs):
+        pass
+
+    def start_session(self, *arg, **kwargs):
+        pass
+
+    def end_session(self, *arg, **kwargs):
+        pass
+
+    def force_shutdown(self, *arg, **kwargs):
+        pass
+
+    def send_stack_trace(self, *arg, **kwargs):
+        pass

ote_cli/ote_cli/utils/tests.py

@@ -72,6 +72,7 @@ def collect_env_vars(work_dir):
 
 def check_run(cmd, **kwargs):
     result = subprocess.run(cmd, stderr=subprocess.PIPE, **kwargs)
+    print(f"***** result.returncode = {result.returncode} *****")
     assert result.returncode == 0, result.stderr.decode("utf=8")
 
 

ote_cli/requirements.txt

@@ -5,3 +5,4 @@ nbmake
 pytest
 pytest-ordering
 hpopt@git+https://github.com/openvinotoolkit/hyper_parameter_optimization@develop
+openvino-telemetry>=2022.1.0

ote_cli/setup.py

@@ -17,17 +17,54 @@
 # and limitations under the License.
 
 import os
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
 
 from setuptools import find_packages, setup
 
+
+def load_module(name: str = "ote_cli/__init__.py"):
+    """Load Python Module.
+
+    Args:
+        name (str, optional): Name of the module to load.
+            Defaults to "ote_cli/__init__.py".
+    """
+    location = str(Path(__file__).parent / name)
+    spec = spec_from_file_location(name=name, location=location)
+    module = module_from_spec(spec)  # type: ignore
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+
+
+def get_ote_cli_version() -> str:
+    """Get version from `ote_cli.__init__`.
+
+    Version is stored in the main __init__ module in `ote_cli`.
+    The varible storing the version is `__version__`. This function
+    reads `__init__` file, checks `__version__ variable and return
+    the value assigned to it.
+
+    Example:
+        >>> # Assume that __version__ = "0.2.6"
+        >>> get_ote_cli_version()
+        "0.2.6"
+
+    Returns:
+        str: `ote_cli` version.
+    """
+    ote_cli = load_module(name="ote_cli/__init__.py")
+    return ote_cli.__version__
+
+
 with open(
     os.path.join(os.path.dirname(__file__), "requirements.txt"), encoding="UTF-8"
 ) as read_file:
     requirements = [requirement.strip() for requirement in read_file]
 
 setup(
     name="ote_cli",
-    version="0.2",
+    version=get_ote_cli_version(),
     packages=find_packages(exclude=("tools",)),
     install_requires=requirements,
     entry_points={

tests/ote_cli/conftest.py

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions
 # and limitations under the License.
 
+import os
+import pytest
+
 from ote_sdk.test_suite.pytest_insertions import *  # noqa #pylint: disable=unused-import
 
 pytest_plugins = get_pytest_plugins_from_ote()
@@ -21,3 +24,47 @@
 
 def pytest_addoption(parser):
     ote_pytest_addoption_insertion(parser)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def manage_tm_config_for_testing():
+    # check file existance both 'isip' and 'openvino_telemetry' if not, create it.
+    # and backup contents if exist
+    cfg_dir = os.path.join(os.path.expanduser("~"), "intel")
+    isip_path = os.path.join(cfg_dir, "isip")
+    otm_path = os.path.join(cfg_dir, "openvino_telemetry")
+    isip_exist = os.path.exists(isip_path)
+    otm_exist = os.path.exists(otm_path)
+
+    isip_backup = None
+    if not isip_exist:
+        with open(isip_path, "w") as f:
+            f.write("0")
+    else:
+        with open(isip_path, "r") as f:
+            isip_backup = f.read()
+
+    otm_backup = None
+    if not otm_exist:
+        with open(otm_path, "w") as f:
+            f.write("0")
+    else:
+        with open(otm_path, "r") as f:
+            otm_backup = f.read()
+
+    yield
+
+    # restore or remove
+    if not isip_exist:
+        os.remove(isip_path)
+    else:
+        if isip_backup is not None:
+            with open(isip_path, "w") as f:
+                f.write(isip_backup)
+
+    if not otm_exist:
+        os.remove(otm_path)
+    else:
+        if otm_backup is not None:
+            with open(otm_path, "w") as f:
+                f.write(otm_backup)