From 7a38a2284a4b7a8a7fb2bc7c9ca12a5813b9287c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Pi=C5=82atowski?= Date: Sun, 21 Nov 2021 17:00:08 +0100 Subject: [PATCH 1/4] Add statistics aggregation --- depthai_demo.py | 34 +++++++++++++- depthai_helpers/metrics.py | 61 +++++++++++++++++++++++++ log_system_information.py | 91 +++++++++++++++++++------------------- requirements-optional.txt | 3 +- requirements.txt | 5 ++- 5 files changed, 144 insertions(+), 50 deletions(-) create mode 100644 depthai_helpers/metrics.py diff --git a/depthai_demo.py b/depthai_demo.py index af55ffd97..b3a94fa33 100755 --- a/depthai_demo.py +++ b/depthai_demo.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import json import os import sys import time @@ -14,6 +15,7 @@ from depthai_helpers.arg_manager import parseArgs from depthai_helpers.config_manager import ConfigManager, DEPTHAI_ZOO, DEPTHAI_VIDEOS +from depthai_helpers.metrics import MetricManager from depthai_helpers.version_check import checkRequirementsVersion from depthai_sdk import FPSHandler, loadModule, getDeviceInfo, downloadYTVideo, Previews, resizeLetterbox from depthai_sdk.managers import NNetManager, PreviewManager, PipelineManager, EncodingManager, BlobManager @@ -58,9 +60,13 @@ def run_all(self, conf): self.setup(conf) self.run() - def __init__(self, displayFrames=True, onNewFrame = noop, onShowFrame = noop, onNn = noop, onReport = noop, onSetup = noop, onTeardown = noop, onIter = noop, shouldRun = lambda: True): + def __init__(self, displayFrames=True, onNewFrame = noop, onShowFrame = noop, onNn = noop, onReport = noop, onSetup = noop, onTeardown = noop, onIter = noop, shouldRun = lambda: True, collectMetrics=False): self._openvinoVersion = None self._displayFrames = displayFrames + if collectMetrics: + self.enableMetrics() + else: + self.metrics = None self.onNewFrame = onNewFrame self.onShowFrame = onShowFrame @@ -89,6 +95,9 @@ def setCallbacks(self, onNewFrame=None, onShowFrame=None, onNn=None, onReport=No if shouldRun is not None: self.shouldRun = shouldRun + def enableMetrics(self): + self.metrics = MetricManager() + def setup(self, conf: ConfigManager): print("Setting up demo...") self._conf = conf @@ -122,6 +131,8 @@ def setup(self, conf: ConfigManager): self._pm.setNnManager(self._nnManager) self._device = dai.Device(self._pm.pipeline.getOpenVINOVersion(), self._deviceInfo, usb2Mode=self._conf.args.usbSpeed == "usb2") + if self.metrics is not None: + self.metrics.reportDevice(self._device) if self._deviceInfo.desc.protocol == dai.XLinkProtocol.X_LINK_USB_VSC: print("USB Connection speed: {}".format(self._device.getUsbSpeed())) self._conf.adjustParamsToDevice(self._device) @@ -618,7 +629,28 @@ def showError(self, error): msgBox.setStandardButtons(QMessageBox.Ok) msgBox.exec() + def setupDataCollection(self): + try: + with Path(".consent").open() as f: + accepted = json.load(f)["statistics"] + except: + msgBox = QMessageBox() + msgBox.setIcon(QMessageBox.Information) + msgBox.setText("Can we collect your device and runtime statistics? \nThese will help us improve the tools you and other users use, including this demo") + msgBox.setWindowTitle("Statistics consent") + msgBox.setStandardButtons(QMessageBox.No | QMessageBox.Yes) + msgBox.setDefaultButton(QMessageBox.Yes) + accepted = msgBox.exec() == QMessageBox.Yes + try: + with Path('.consent').open('w') as f: + json.dump({"statistics": accepted}, f) + except: + pass + if accepted: + self._demoInstance.enableMetrics() + def start(self): + self.setupDataCollection() self.running = True self.worker = Worker(self._demoInstance, parent=self, conf=self.confManager, selectedPreview=self.selectedPreview) self.worker.signals.updatePreviewSignal.connect(self.updatePreview) diff --git a/depthai_helpers/metrics.py b/depthai_helpers/metrics.py new file mode 100644 index 000000000..2c7cad691 --- /dev/null +++ b/depthai_helpers/metrics.py @@ -0,0 +1,61 @@ +from datetime import datetime + +import pyrebase +import depthai as dai + +from log_system_information import make_sys_report + + +class MetricManager: + def __init__(self): + self.config = { + "apiKey": "AIzaSyDCrQs4SYUXZiN1qASxaiMU33YKSImp6kw", + "authDomain": "depthai-data.firebaseapp.com", + "databaseURL": "https://depthai-data-default-rtdb.firebaseio.com/", + "storageBucket": "depthai-data.appspot.com" + } + + self.firebase = pyrebase.initialize_app(self.config) + self.db = self.firebase.database() + self.demo_table = self.db.child("demo") + + def reportDevice(self, device: dai.Device): + try: + device_info = device.getDeviceInfo() + mxid = device_info.getMxId() + except: + return + try: + cameras = list(map(lambda camera: camera.name, device.getConnectedCameras())) + except: + cameras = [] + try: + usb = device.getUsbSpeed().name + except: + usb = "None" + sys_report = make_sys_report() + data = { + "mxid": mxid, + "timestamp": datetime.utcnow().isoformat(), + "system": sys_report, + "api": { + "location": dai.__file__, + "version": dai.__version__ + }, + "device": { + "cameras": cameras, + "state": device_info.state.name, + "name": device_info.desc.name, + "platform": device_info.desc.platform.name, + "protocol": device_info.desc.protocol.name, + "usb": usb + } + } + self.demo_table.update({ + f"devices/{mxid}": data + }) + +if __name__ == "__main__": + mm = MetricManager() + with dai.Device() as device: + mm.reportDevice(device) \ No newline at end of file diff --git a/log_system_information.py b/log_system_information.py index a666ebb19..944470212 100755 --- a/log_system_information.py +++ b/log_system_information.py @@ -1,53 +1,52 @@ #!/usr/bin/env python3 import json -import subprocess -import sys from pip._internal.operations.freeze import freeze import platform -try: - import usb.core -except ImportError: - subprocess.check_call([sys.executable, "-m", "pip", "install", "pyusb"]) - import usb.core - -def get_usb(): - speeds = ["Unknown", "Low", "Full", "High", "Super", "SuperPlus"] - format_hex = lambda val: f"{val:#0{6}x}" - try: - for dev in usb.core.find(find_all=True): - yield { - "port": dev.port_number, - "vendor_id": format_hex(dev.idVendor), - "product_id": format_hex(dev.idProduct), - "speed": speeds[dev.speed] if dev.speed < len(speeds) else dev.speed - } - except usb.core.NoBackendError: - yield "No USB backend found" - - -data = { - "architecture": ' '.join(platform.architecture()).strip(), - "machine": platform.machine(), - "platform": platform.platform(), - "processor": platform.processor(), - "python_build": ' '.join(platform.python_build()).strip(), - "python_compiler": platform.python_compiler(), - "python_implementation": platform.python_implementation(), - "python_version": platform.python_version(), - "release": platform.release(), - "system": platform.system(), - "version": platform.version(), - "win32_ver": ' '.join(platform.win32_ver()).strip(), - "uname": ' '.join(platform.uname()).strip(), - "packages": list(freeze(local_only=True)), - "usb": list(get_usb()), -} - -with open("log_system_information.json", "w") as f: - json.dump(data, f, indent=4) - -print(json.dumps(data, indent=4)) -print("System info gathered successfully - saved as \"log_system_information.json\"") +def make_sys_report(): + def get_usb(): + try: + import usb.core + except ImportError: + return "NoLib" + speeds = ["Unknown", "Low", "Full", "High", "Super", "SuperPlus"] + format_hex = lambda val: f"{val:#0{6}x}" + try: + for dev in usb.core.find(find_all=True): + yield { + "port": dev.port_number, + "vendor_id": format_hex(dev.idVendor), + "product_id": format_hex(dev.idProduct), + "speed": speeds[dev.speed] if dev.speed < len(speeds) else dev.speed + } + except usb.core.NoBackendError: + yield "No USB backend found" + + return { + "architecture": ' '.join(platform.architecture()).strip(), + "machine": platform.machine(), + "platform": platform.platform(), + "processor": platform.processor(), + "python_build": ' '.join(platform.python_build()).strip(), + "python_compiler": platform.python_compiler(), + "python_implementation": platform.python_implementation(), + "python_version": platform.python_version(), + "release": platform.release(), + "system": platform.system(), + "version": platform.version(), + "win32_ver": ' '.join(platform.win32_ver()).strip(), + "uname": ' '.join(platform.uname()).strip(), + "packages": list(freeze(local_only=True)), + "usb": list(get_usb()), + } + + +if __name__ == "__main__": + data = make_sys_report() + with open("log_system_information.json", "w") as f: + json.dump(data, f, indent=4) + + print(json.dumps(data, indent=4)) + print("System info gathered successfully - saved as \"log_system_information.json\"") diff --git a/requirements-optional.txt b/requirements-optional.txt index fb04db6cb..3fd792acd 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,2 +1,3 @@ open3d==0.10.0.0; platform_machine != "armv6l" and platform_machine != "armv7l" and python_version < "3.9" -ffmpy3==0.2.4 \ No newline at end of file +ffmpy3==0.2.4 +pyusb \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 46237a6a7..b1f65b421 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests==2.24.0 +requests==2.11.1 argcomplete==1.12.1 opencv-python==4.5.4.58 ; platform_machine != "aarch64" and platform_machine != "armv6l" and platform_machine != "armv7l" and python_version == "3.10" opencv-python==4.5.1.48 ; platform_machine != "aarch64" and platform_machine != "armv6l" and platform_machine != "armv7l" and python_version != "3.10" @@ -9,4 +9,5 @@ opencv-contrib-python==4.4.0.46 ; platform_machine == "armv6l" or platform_machi -e ./depthai_sdk --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/ depthai==2.11.1.0.dev+dfd52ac01c3b7514e85cb894b9d5381e999859df -PySide6==6.2.0 \ No newline at end of file +pyside6==6.2.0 +pyrebase==3.0.27 \ No newline at end of file From fb6e6c2e697443ec5f08e6e4cf79bb0ebeb42ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Pi=C5=82atowski?= Date: Sun, 21 Nov 2021 17:19:22 +0100 Subject: [PATCH 2/4] add anonimization --- depthai_demo.py | 2 +- depthai_helpers/metrics.py | 3 +-- log_system_information.py | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/depthai_demo.py b/depthai_demo.py index b3a94fa33..71000c8db 100755 --- a/depthai_demo.py +++ b/depthai_demo.py @@ -636,7 +636,7 @@ def setupDataCollection(self): except: msgBox = QMessageBox() msgBox.setIcon(QMessageBox.Information) - msgBox.setText("Can we collect your device and runtime statistics? \nThese will help us improve the tools you and other users use, including this demo") + msgBox.setText("Can we collect anonymous device and runtime statistics? \nThese will help us improve the tools you and other users use, including this demo") msgBox.setWindowTitle("Statistics consent") msgBox.setStandardButtons(QMessageBox.No | QMessageBox.Yes) msgBox.setDefaultButton(QMessageBox.Yes) diff --git a/depthai_helpers/metrics.py b/depthai_helpers/metrics.py index 2c7cad691..ba609269f 100644 --- a/depthai_helpers/metrics.py +++ b/depthai_helpers/metrics.py @@ -33,13 +33,12 @@ def reportDevice(self, device: dai.Device): usb = device.getUsbSpeed().name except: usb = "None" - sys_report = make_sys_report() + sys_report = make_sys_report(anonymous=True) data = { "mxid": mxid, "timestamp": datetime.utcnow().isoformat(), "system": sys_report, "api": { - "location": dai.__file__, "version": dai.__version__ }, "device": { diff --git a/log_system_information.py b/log_system_information.py index 944470212..422d1a6ba 100755 --- a/log_system_information.py +++ b/log_system_information.py @@ -5,7 +5,7 @@ import platform -def make_sys_report(): +def make_sys_report(anonymous=False): def get_usb(): try: import usb.core @@ -37,7 +37,7 @@ def get_usb(): "system": platform.system(), "version": platform.version(), "win32_ver": ' '.join(platform.win32_ver()).strip(), - "uname": ' '.join(platform.uname()).strip(), + "uname": ' '.join(platform.uname()).strip() if not anonymous else "hidden", "packages": list(freeze(local_only=True)), "usb": list(get_usb()), } From dc56b3673f99ce476558fca353b7cb5a1254778b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Pi=C5=82atowski?= Date: Mon, 22 Nov 2021 15:23:29 +0100 Subject: [PATCH 3/4] change consent to opt-out --- depthai_demo.py | 38 +++++++++++++++++------------------- depthai_helpers/metrics.py | 2 +- gui/main.py | 4 ++++ gui/views/MiscProperties.qml | 32 ++++++++++++++++++++++++++++-- gui/views/root.qml | 1 + log_system_information.py | 15 +++++++++----- requirements-optional.txt | 2 +- 7 files changed, 65 insertions(+), 29 deletions(-) diff --git a/depthai_demo.py b/depthai_demo.py index 71000c8db..5e604e0f4 100755 --- a/depthai_demo.py +++ b/depthai_demo.py @@ -63,10 +63,7 @@ def run_all(self, conf): def __init__(self, displayFrames=True, onNewFrame = noop, onShowFrame = noop, onNn = noop, onReport = noop, onSetup = noop, onTeardown = noop, onIter = noop, shouldRun = lambda: True, collectMetrics=False): self._openvinoVersion = None self._displayFrames = displayFrames - if collectMetrics: - self.enableMetrics() - else: - self.metrics = None + self.toggleMetrics(collectMetrics) self.onNewFrame = onNewFrame self.onShowFrame = onShowFrame @@ -95,8 +92,11 @@ def setCallbacks(self, onNewFrame=None, onShowFrame=None, onNn=None, onReport=No if shouldRun is not None: self.shouldRun = shouldRun - def enableMetrics(self): - self.metrics = MetricManager() + def toggleMetrics(self, enabled): + if enabled: + self.metrics = MetricManager() + else: + self.metrics = None def setup(self, conf: ConfigManager): print("Setting up demo...") @@ -592,6 +592,7 @@ def onSetup(self, instance): else: self.signals.setDataSignal.emit(["countLabels", []]) self.signals.setDataSignal.emit(["depthEnabled", self.conf.useDepth]) + self.signals.setDataSignal.emit(["statisticsAccepted", self.instance.metrics is not None]) self.signals.setDataSignal.emit(["modelChoices", sorted(self.conf.getAvailableZooModels(), key=cmp_to_key(lambda a, b: -1 if a == "mobilenet-ssd" else 1 if b == "mobilenet-ssd" else -1 if a < b else 1))]) @@ -634,20 +635,9 @@ def setupDataCollection(self): with Path(".consent").open() as f: accepted = json.load(f)["statistics"] except: - msgBox = QMessageBox() - msgBox.setIcon(QMessageBox.Information) - msgBox.setText("Can we collect anonymous device and runtime statistics? \nThese will help us improve the tools you and other users use, including this demo") - msgBox.setWindowTitle("Statistics consent") - msgBox.setStandardButtons(QMessageBox.No | QMessageBox.Yes) - msgBox.setDefaultButton(QMessageBox.Yes) - accepted = msgBox.exec() == QMessageBox.Yes - try: - with Path('.consent').open('w') as f: - json.dump({"statistics": accepted}, f) - except: - pass - if accepted: - self._demoInstance.enableMetrics() + accepted = True + + self._demoInstance.toggleMetrics(accepted) def start(self): self.setupDataCollection() @@ -787,6 +777,14 @@ def guiOnReloadDevices(self): if len(devices) > 0: self.worker.signals.setDataSignal.emit(["restartRequired", True]) + def guiOnStaticticsConsent(self, value): + try: + with Path('.consent').open('w') as f: + json.dump({"statistics": value}, f) + except: + pass + self.worker.signals.setDataSignal.emit(["restartRequired", True]) + def guiOnToggleColorEncoding(self, enabled, fps): oldConfig = self.confManager.args.encode or {} if enabled: diff --git a/depthai_helpers/metrics.py b/depthai_helpers/metrics.py index ba609269f..f3035d9ca 100644 --- a/depthai_helpers/metrics.py +++ b/depthai_helpers/metrics.py @@ -33,7 +33,7 @@ def reportDevice(self, device: dai.Device): usb = device.getUsbSpeed().name except: usb = "None" - sys_report = make_sys_report(anonymous=True) + sys_report = make_sys_report(anonymous=True, skipUsb=True) data = { "mxid": mxid, "timestamp": datetime.utcnow().isoformat(), diff --git a/gui/main.py b/gui/main.py index f53a0ad66..ec8179240 100644 --- a/gui/main.py +++ b/gui/main.py @@ -89,6 +89,10 @@ def applyAndRestart(self): def reloadDevices(self): DemoQtGui.instance.guiOnReloadDevices() + @Slot(bool) + def toggleStatisticsConsent(self, value): + DemoQtGui.instance.guiOnStaticticsConsent(value) + @Slot(str) def selectDevice(self, value): DemoQtGui.instance.guiOnSelectDevice(value) diff --git a/gui/views/MiscProperties.qml b/gui/views/MiscProperties.qml index b3c82ca0d..1798bf188 100644 --- a/gui/views/MiscProperties.qml +++ b/gui/views/MiscProperties.qml @@ -176,7 +176,7 @@ ListView { id: textField3 x: 110 y: 352 - width: 227 + width: 170 height: 27 bottomPadding: 7 placeholderText: qsTr("/path/to/report.csv") @@ -203,7 +203,7 @@ ListView { id: textField4 x: 116 y: 150 - width: 227 + width: 170 height: 27 bottomPadding: 7 placeholderText: qsTr("/path/to/output/directory/") @@ -225,6 +225,34 @@ ListView { verticalAlignment: Text.AlignVCenter font.family: "Courier" } + + Text { + id: textOptions + x: 350 + y: 8 + width: 185 + height: 30 + color: "#ffffff" + text: qsTr("Demo options") + font.pixelSize: 26 + horizontalAlignment: Text.AlignHCenter + font.family: "Courier" + font.styleName: "Regular" + } + + Switch { + id: consentSwitch + x: 350 + y: 40 + checked: statisticsAccepted + width: 250 + height: 27 + text: qsTr("Send anonymous usage data") + bottomPadding: 5 + onToggled: { + appBridge.toggleStatisticsConsent(consentSwitch.checked) + } + } } } /*##^## diff --git a/gui/views/root.qml b/gui/views/root.qml index 847b7cf29..50d4cdabc 100644 --- a/gui/views/root.qml +++ b/gui/views/root.qml @@ -75,6 +75,7 @@ ApplicationWindow { property var restartRequired property var deviceChoices property var depthEnabled: true + property var statisticsAccepted: true AppBridge { id: appBridge diff --git a/log_system_information.py b/log_system_information.py index 422d1a6ba..fe3103be3 100755 --- a/log_system_information.py +++ b/log_system_information.py @@ -5,12 +5,13 @@ import platform -def make_sys_report(anonymous=False): +def make_sys_report(anonymous=False, skipUsb=False): def get_usb(): try: import usb.core except ImportError: - return "NoLib" + yield "NoLib" + return speeds = ["Unknown", "Low", "Full", "High", "Super", "SuperPlus"] format_hex = lambda val: f"{val:#0{6}x}" try: @@ -24,7 +25,7 @@ def get_usb(): except usb.core.NoBackendError: yield "No USB backend found" - return { + result = { "architecture": ' '.join(platform.architecture()).strip(), "machine": platform.machine(), "platform": platform.platform(), @@ -37,11 +38,15 @@ def get_usb(): "system": platform.system(), "version": platform.version(), "win32_ver": ' '.join(platform.win32_ver()).strip(), - "uname": ' '.join(platform.uname()).strip() if not anonymous else "hidden", "packages": list(freeze(local_only=True)), - "usb": list(get_usb()), } + if not skipUsb: + result["usb"] = list(get_usb()) + if not anonymous: + result["uname"] = ' '.join(platform.uname()).strip(), + return result + if __name__ == "__main__": data = make_sys_report() diff --git a/requirements-optional.txt b/requirements-optional.txt index 3fd792acd..86f13b575 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,3 +1,3 @@ open3d==0.10.0.0; platform_machine != "armv6l" and platform_machine != "armv7l" and python_version < "3.9" ffmpy3==0.2.4 -pyusb \ No newline at end of file +pyusb==1.2.1 \ No newline at end of file From 22fa50ee1dfff42e93e500f93cfa0f4198d287b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Pi=C5=82atowski?= Date: Mon, 22 Nov 2021 19:24:24 +0100 Subject: [PATCH 4/4] Update README.md --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 1bbb6eda3..f65d8add7 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,16 @@ with the help of the `myriad_compile` tool. When producing blobs, the following Example of command execution: /deployment_tools/inference_engine/lib/intel64/myriad_compile -m ./ResNet50.xml -o ResNet50.blob -ip U8 -VPU_NUMBER_OF_SHAVES 4 -VPU_NUMBER_OF_CMX_SLICES 4 + +## Usage statistics + +By default, the demo script will collect anonymous usage statistics during runtime. These include: +- Device-specific information (like mxid, connected cameras, device state and connection type) +- Environment-specific information (like OS type, python version, package versions) + +We gather this data to better understand what environemnts are our users using, as well as assist better in support questions. + +**All of the data we collect is anonymous and you can disable it at any time**. To do so, click on the "Misc" tab and disable sending the statistics. ## Reporting issues