diff --git a/onadata/libs/serializers/data_serializer.py b/onadata/libs/serializers/data_serializer.py index 4f84cc44cf..352eaff3d0 100644 --- a/onadata/libs/serializers/data_serializer.py +++ b/onadata/libs/serializers/data_serializer.py @@ -203,6 +203,7 @@ def validate(self, attrs): properties={ 'submitted_by': 'user', 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', 'organization': 'xform__user__profile__organization'}, additional_context={'from': 'XML Submissions'} ) @@ -318,6 +319,7 @@ def validate(self, attrs): properties={ 'submitted_by': 'user', 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', 'organization': 'xform__user__profile__organization'}, additional_context={'from': 'JSON Submission'} ) @@ -347,7 +349,11 @@ class RapidProSubmissionSerializer(BaseRapidProSubmissionSerializer): """ @track_object_event( user_field='xform__user', - properties={'submitted_by': 'user', 'xform_id': 'xform__pk'}, + properties={ + 'submitted_by': 'user', + 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', + }, additional_context={'from': 'RapidPro'} ) def create(self, validated_data): @@ -369,7 +375,11 @@ class RapidProJSONSubmissionSerializer(BaseRapidProSubmissionSerializer): """ @track_object_event( user_field='xform__user', - properties={'submitted_by': 'user', 'xform_id': 'xform__pk'}, + properties={ + 'submitted_by': 'user', + 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', + }, additional_context={'from': 'RapidPro(JSON)'} ) def create(self, validated_data): @@ -392,7 +402,11 @@ class FLOIPListSerializer(serializers.ListSerializer): """ @track_object_event( user_field='xform__user', - properties={'submitted_by': 'user', 'xform_id': 'xform__pk'}, + properties={ + 'submitted_by': 'user', + 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', + }, additional_context={'from': 'FLOIP'} ) def create(self, validated_data): diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index da8d35a4c1..d9b00c357d 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -22,6 +22,7 @@ DataViewMinimalSerializer from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.tag_list_serializer import TagListSerializer +from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.cache_tools import ( PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, PROJ_PERM_CACHE, PROJ_SUB_DATE_CACHE, PROJ_TEAM_USERS_CACHE, @@ -438,6 +439,13 @@ def update(self, instance, validated_data): return instance + @track_object_event( + user_field='created_by', + properties={ + 'created_by': 'created_by', + 'project_id': 'pk', + 'project_name': 'name'} + ) def create(self, validated_data): metadata = validated_data.get('metadata', dict()) if metadata is None: diff --git a/onadata/libs/serializers/user_profile_serializer.py b/onadata/libs/serializers/user_profile_serializer.py index 5dcce2d877..f2686c4131 100644 --- a/onadata/libs/serializers/user_profile_serializer.py +++ b/onadata/libs/serializers/user_profile_serializer.py @@ -29,6 +29,7 @@ from onadata.libs.permissions import CAN_VIEW_PROFILE, is_organization from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.utils.cache_tools import IS_ORG +from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.email import ( get_verification_url, get_verification_email_data ) @@ -229,6 +230,11 @@ def update(self, instance, validated_data): return super(UserProfileSerializer, self).update(instance, params) + @track_object_event( + user_field='user', + properties={ + 'name': 'name', + 'country': 'country'}) def create(self, validated_data): params = validated_data request = self.context.get('request') diff --git a/onadata/libs/tests/utils/test_analytics.py b/onadata/libs/tests/utils/test_analytics.py index 2465efcd08..930574a959 100644 --- a/onadata/libs/tests/utils/test_analytics.py +++ b/onadata/libs/tests/utils/test_analytics.py @@ -7,6 +7,7 @@ from django.contrib.auth.models import User, AnonymousUser from django.core.files.uploadedfile import InMemoryUploadedFile +from django.conf import settings from django.test import override_settings import onadata.libs.utils.analytics @@ -14,7 +15,7 @@ TestAbstractViewSet from onadata.apps.api.viewsets.xform_submission_viewset import \ XFormSubmissionViewSet -from onadata.libs.utils.analytics import get_user_id, sanitize_metric_values +from onadata.libs.utils.analytics import get_user_id class TestAnalytics(TestAbstractViewSet): @@ -31,7 +32,7 @@ def test_get_user_id(self): self.assertTrue(len(user2.email) > 0) self.assertEqual(get_user_id(user2), user2.email) - @override_settings(SEGMENT_WRITE_KEY='123', HOSTNAME='test-server') + @override_settings(SEGMENT_WRITE_KEY='123') def test_track(self): """Test analytics.track() function. """ @@ -46,55 +47,39 @@ def test_track(self): user1.username, 'testing track function', {'value': 1}, - {'source': 'test-server'}) - - def test_sanitize_metric_values(self): - """Test sanitize_metric_values()""" - expected_output = { - 'action_from': 'XML_Submissions', - 'event_by': 'bob', - 'ip': '18.203.134.101', - 'path': '/enketo/1814/submission', - 'source': 'onadata-ona-stage', - 'url': 'https://stage-api.ona.io/enketo/1814/submission', - 'userId': 1, - 'xform_id': 1} - context = { - 'action_from': 'XML Submissions', - 'event_by': 'bob', - 'ip': '18.203.134.101', - 'library': { - 'name': 'analytics-python', - 'version': '1.2.9' - }, - 'organization': '', - 'path': '/enketo/1814/submission', - 'source': 'onadata-ona-stage', - 'url': 'https://stage-api.ona.io/enketo/1814/submission', - 'userId': 1, - 'xform_id': 1} - - self.assertEqual(sanitize_metric_values(context), expected_output) + {'page': {}, 'campaign': {}, 'active': True}) @override_settings( - SEGMENT_WRITE_KEY='123', HOSTNAME='test-server', - APPOPTICS_API_TOKEN='123') + SEGMENT_WRITE_KEY='123') def test_submission_tracking(self): """Test that submissions are tracked""" segment_mock = MagicMock() - appoptics_mock = MagicMock() onadata.libs.utils.analytics.segment_analytics = segment_mock onadata.libs.utils.analytics.init_analytics() self.assertEqual(segment_mock.write_key, '123') # Test out that the track_object_event decorator - # Tracks created submissions + # Tracks created submissions, XForms and Projects view = XFormSubmissionViewSet.as_view({ 'post': 'create', 'head': 'create' }) self._publish_xls_form_to_project() - onadata.libs.utils.analytics.appoptics_api = appoptics_mock + segment_mock.track.assert_called_with( + 'bob@columbia.edu', + 'XForm created', + { + 'created_by': self.xform.user, + 'xform_id': self.xform.pk, + 'xform_name': self.xform.title, + 'from': 'Publish XLS Form', + 'value': 1 + }, + { + 'page': {}, + 'campaign': {}, + 'active': True + }) s = self.surveys[0] media_file = "1335783522563.jpg" path = os.path.join(self.main_directory, 'fixtures', @@ -110,6 +95,13 @@ def test_submission_tracking(self): data = {'xml_submission_file': sf, 'media_file': f} request = self.factory.post(request_path, data) request.user = AnonymousUser() + request.META['HTTP_DATE'] = '2020-09-10T11:56:32.424726+00:00' + request.META['HTTP_REFERER'] = settings.HOSTNAME +\ + ':8000' + request.META['HTTP_USER_AGENT'] =\ + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit'\ + '/537.36 (KHTML, like Gecko) Chrome'\ + '/83.0.4103.61 Safari/537.36' response = view(request, username=self.user.username) self.assertContains(response, 'Successful submission', status_code=201) @@ -128,35 +120,25 @@ def test_submission_tracking(self): 'Submission created', { 'xform_id': self.xform.pk, + 'project_id': self.xform.project.pk, 'organization': 'Bob Inc.', - 'from': 'XML Submissions', + 'from': 'Submission collected from Enketo', 'label': f'form-{form_id}-owned-by-{username}', 'value': 1, 'event_by': 'anonymous' }, - { - 'source': 'test-server', - 'organization': 'Bob Inc.', - 'event_by': 'anonymous', - 'action_from': 'XML Submissions', - 'xform_id': self.xform.pk, - 'path': f'/{username}/submission', - 'url': f'http://testserver/{username}/submission', - 'ip': '127.0.0.1', - 'userId': self.user.id - }) - - appoptics_mock.submit_measurement.assert_called_with( - 'Submission created', - 1, - tags={ - 'source': 'test-server', - 'event_by': 'anonymous', - 'organization': 'Bob_Inc.', - 'action_from': 'XML_Submissions', - 'xform_id': self.xform.pk, - 'path': f'/{username}/submission', - 'url': f'http://testserver/{username}/submission', + {'page': { + 'path': '/bob/submission', + 'referrer': settings.HOSTNAME + ':8000', + 'url': 'http://testserver/bob/submission' + }, + 'campaign': { + 'source': settings.HOSTNAME}, + 'active': True, 'ip': '127.0.0.1', - 'userId': self.user.id - }) + 'userId': self.xform.user.pk, + 'receivedAt': '2020-09-10T11:56:32.424726+00:00', + 'userAgent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit' + '/537.36 (KHTML, like Gecko) Chrome' + '/83.0.4103.61 Safari/537.36'} + ) diff --git a/onadata/libs/utils/analytics.py b/onadata/libs/utils/analytics.py index 0812b10af0..4a29c6c338 100644 --- a/onadata/libs/utils/analytics.py +++ b/onadata/libs/utils/analytics.py @@ -1,63 +1,33 @@ # -*- coding: utf-8 -*- # Analytics package for tracking and measuring with services like Segment. # Heavily borrowed from RapidPro's temba.utils.analytics -import sys - import analytics as segment_analytics -from typing import Dict, List, Optional, Any -import appoptics_metrics -from appoptics_metrics import sanitize_metric_name, exceptions +from typing import Dict, List, Optional from django.conf import settings from django.contrib.auth.models import User from django.utils import timezone -from onadata.apps.logger.models import Instance +from onadata.apps.logger.models import Instance, XForm, Project +from onadata.apps.main.models import UserProfile from onadata.libs.utils.common_tags import ( - INSTANCE_CREATE_EVENT, INSTANCE_UPDATE_EVENT) -from onadata.libs.utils.common_tools import report_exception - + INSTANCE_CREATE_EVENT, + INSTANCE_UPDATE_EVENT, + XFORM_CREATION_EVENT, + PROJECT_CREATION_EVENT, + USER_CREATION_EVENT) -appoptics_api = None _segment = False def init_analytics(): """Initialize the analytics agents with write credentials.""" segment_write_key = getattr(settings, 'SEGMENT_WRITE_KEY', None) - appoptics_api_token = getattr(settings, "APPOPTICS_API_TOKEN", None) if segment_write_key: global _segment segment_analytics.write_key = segment_write_key _segment = True - if appoptics_api_token: - global appoptics_api - appoptics_api = appoptics_metrics.connect( - appoptics_api_token, sanitizer=sanitize_metric_name) - - -def sanitize_metric_values(data: Dict[str, Any]) -> Dict[str, Any]: - """ - Sanitizes values in order to ensure the values are valid - for AppOptics - """ - sanitized_data = data.copy() - for key, value in data.items(): - new_value = value - if new_value and not isinstance(new_value, dict): - if isinstance(new_value, str): - if ' ' in new_value: - new_value = new_value.replace(' ', '_') - - if '(' in value or ')' in new_value: - new_value = new_value.replace(')', "").replace('(', "") - - sanitized_data.update({key: new_value}) - else: - sanitized_data.pop(key) - return sanitized_data - def get_user_id(user): """Return a user email or username or the string 'anonymous'""" @@ -133,6 +103,12 @@ def get_event_name(self) -> str: event_name = INSTANCE_UPDATE_EVENT else: event_name = INSTANCE_CREATE_EVENT + elif isinstance(self.tracked_obj, XForm) and not event_name: + event_name = XFORM_CREATION_EVENT + elif isinstance(self.tracked_obj, Project): + event_name = PROJECT_CREATION_EVENT + elif isinstance(self.tracked_obj, UserProfile): + event_name = USER_CREATION_EVENT return event_name def get_event_label(self) -> str: @@ -143,12 +119,33 @@ def get_event_label(self) -> str: event_label = f"form-{form_id}-owned-by-{username}" return event_label + def get_request_origin(self, request, tracking_properties): + if isinstance(self.tracked_obj, Instance): + try: + user_agent = request.META['HTTP_USER_AGENT'] + if 'Android' in user_agent: + event_source = 'Submission collected from ODK COLLECT' + elif 'Chrome' or 'Mozilla' or 'Safari' in user_agent: + event_source = 'Submission collected from Enketo' + except KeyError: + event_source = "" + else: + event_source = "" + tracking_properties.update({'from': event_source}) + return tracking_properties + def _track_object_event(self, obj, request=None) -> None: self.tracked_obj = obj self.set_user() event_name = self.get_event_name() label = self.get_event_label() tracking_properties = self.get_tracking_properties(label=label) + try: + if tracking_properties['from'] == 'XML Submissions': + tracking_properties = self.get_request_origin( + request, tracking_properties) + except KeyError: + pass track( self.user, event_name, properties=tracking_properties, request=request) @@ -171,13 +168,13 @@ def decorator(obj, *args): def track(user, event_name, properties=None, context=None, request=None): """Record actions with the track() API call to the analytics agents.""" - if _segment or appoptics_api: - context = context or {} - context['source'] = settings.HOSTNAME - - properties = properties or {} - + if _segment: user_id = get_user_id(user) + properties = properties or {} + context = context or {} + # Introduce inner page and campaign object within the context + context['page'] = {} + context['campaign'] = {} if 'value' not in properties: properties['value'] = 1 @@ -186,35 +183,18 @@ def track(user, event_name, properties=None, context=None, request=None): submitted_by_user = properties.pop('submitted_by') submitted_by = get_user_id(submitted_by_user) properties['event_by'] = submitted_by - if submitted_by_user: - context['event_by'] = submitted_by_user.username - else: - context['event_by'] = submitted_by - - if 'organization' in properties: - context['organization'] = properties.get('organization') - - if 'from' in properties: - context['action_from'] = properties.get('from') - if 'xform_id' in properties: - context['xform_id'] = properties['xform_id'] + context['active'] = True if request: - context['path'] = request.path - context['url'] = request.build_absolute_uri() context['ip'] = request.META.get('REMOTE_ADDR', '') context['userId'] = user.id + context['receivedAt'] = request.META.get('HTTP_DATE', '') + context['userAgent'] = request.META.get('HTTP_USER_AGENT', '') + context['campaign']['source'] = settings.HOSTNAME + context['page']['path'] = request.path + context['page']['referrer'] = request.META.get('HTTP_REFERER', '') + context['page']['url'] = request.build_absolute_uri() if _segment: segment_analytics.track(user_id, event_name, properties, context) - - if appoptics_api: - tags = sanitize_metric_values(context) - try: - appoptics_api.submit_measurement( - event_name, - properties['value'], - tags=tags) - except exceptions.BadRequest as e: - report_exception("Bad AppOptics Request", e, sys.exc_info()) diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index 69dd7e87e5..775c0a1ee4 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -185,3 +185,6 @@ INSTANCE_CREATE_EVENT = 'Submission created' INSTANCE_UPDATE_EVENT = 'Submission updated' +XFORM_CREATION_EVENT = 'XForm created' +PROJECT_CREATION_EVENT = 'Project created' +USER_CREATION_EVENT = 'User account created' diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 97980e71ae..86f714f7f2 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -466,6 +466,7 @@ def submit_csv(username, xform, csv_file, overwrite=False): event_name = None tracking_properties = { 'xform_id': xform.pk, + 'project_id': xform.project.pk, 'submitted_by': event_by, 'label': f'csv-import-for-form-{xform.pk}', 'from': 'CSV Import', diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 7403892d1e..1557e53f85 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -48,6 +48,7 @@ from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.apps.viewer.models.parsed_instance import ParsedInstance from onadata.apps.viewer.signals import process_submission +from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.common_tags import METADATA_FIELDS from onadata.libs.utils.common_tools import report_exception, get_uuid from onadata.libs.utils.model_tools import set_uuid @@ -634,6 +635,14 @@ def publish_form(callback): return {'type': 'alert-error', 'text': text(e)} +@track_object_event( + user_field='user', + properties={ + 'created_by': 'user', + 'xform_id': 'pk', + 'xform_name': 'title'}, + additional_context={'from': 'Publish XLS Form'} +) @transaction.atomic() def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): """Create or update DataDictionary with xls_file, user @@ -655,6 +664,14 @@ def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): project=project) +@track_object_event( + user_field='user', + properties={ + 'created_by': 'user', + 'xform_id': 'pk', + 'xform_name': 'title'}, + additional_context={'from': 'Publish XML Form'} +) def publish_xml_form(xml_file, user, project, id_string=None, created_by=None): xml = xml_file.read() if isinstance(xml, bytes):