From b5b2893f3e595d58a29f218799eb2eca5ae967b5 Mon Sep 17 00:00:00 2001 From: WinnyTroy Date: Thu, 10 Sep 2020 12:58:24 +0300 Subject: [PATCH 1/3] Introduce collecting project information for every submission created Reconstruct the context object being sent to segment Updated tests --- onadata/libs/serializers/data_serializer.py | 20 ++++++-- onadata/libs/tests/utils/test_analytics.py | 52 ++++++++++++--------- onadata/libs/utils/analytics.py | 50 ++++++++++++-------- onadata/libs/utils/csv_import.py | 1 + 4 files changed, 79 insertions(+), 44 deletions(-) diff --git a/onadata/libs/serializers/data_serializer.py b/onadata/libs/serializers/data_serializer.py index 4f84cc44cf..352eaff3d0 100644 --- a/onadata/libs/serializers/data_serializer.py +++ b/onadata/libs/serializers/data_serializer.py @@ -203,6 +203,7 @@ def validate(self, attrs): properties={ 'submitted_by': 'user', 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', 'organization': 'xform__user__profile__organization'}, additional_context={'from': 'XML Submissions'} ) @@ -318,6 +319,7 @@ def validate(self, attrs): properties={ 'submitted_by': 'user', 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', 'organization': 'xform__user__profile__organization'}, additional_context={'from': 'JSON Submission'} ) @@ -347,7 +349,11 @@ class RapidProSubmissionSerializer(BaseRapidProSubmissionSerializer): """ @track_object_event( user_field='xform__user', - properties={'submitted_by': 'user', 'xform_id': 'xform__pk'}, + properties={ + 'submitted_by': 'user', + 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', + }, additional_context={'from': 'RapidPro'} ) def create(self, validated_data): @@ -369,7 +375,11 @@ class RapidProJSONSubmissionSerializer(BaseRapidProSubmissionSerializer): """ @track_object_event( user_field='xform__user', - properties={'submitted_by': 'user', 'xform_id': 'xform__pk'}, + properties={ + 'submitted_by': 'user', + 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', + }, additional_context={'from': 'RapidPro(JSON)'} ) def create(self, validated_data): @@ -392,7 +402,11 @@ class FLOIPListSerializer(serializers.ListSerializer): """ @track_object_event( user_field='xform__user', - properties={'submitted_by': 'user', 'xform_id': 'xform__pk'}, + properties={ + 'submitted_by': 'user', + 'xform_id': 'xform__pk', + 'project_id': 'xform__project__pk', + }, additional_context={'from': 'FLOIP'} ) def create(self, validated_data): diff --git a/onadata/libs/tests/utils/test_analytics.py b/onadata/libs/tests/utils/test_analytics.py index 2465efcd08..137045d06e 100644 --- a/onadata/libs/tests/utils/test_analytics.py +++ b/onadata/libs/tests/utils/test_analytics.py @@ -7,6 +7,7 @@ from django.contrib.auth.models import User, AnonymousUser from django.core.files.uploadedfile import InMemoryUploadedFile +from django.conf import settings from django.test import override_settings import onadata.libs.utils.analytics @@ -31,7 +32,7 @@ def test_get_user_id(self): self.assertTrue(len(user2.email) > 0) self.assertEqual(get_user_id(user2), user2.email) - @override_settings(SEGMENT_WRITE_KEY='123', HOSTNAME='test-server') + @override_settings(SEGMENT_WRITE_KEY='123') def test_track(self): """Test analytics.track() function. """ @@ -46,7 +47,7 @@ def test_track(self): user1.username, 'testing track function', {'value': 1}, - {'source': 'test-server'}) + {'page': {}, 'active': True}) def test_sanitize_metric_values(self): """Test sanitize_metric_values()""" @@ -77,7 +78,7 @@ def test_sanitize_metric_values(self): self.assertEqual(sanitize_metric_values(context), expected_output) @override_settings( - SEGMENT_WRITE_KEY='123', HOSTNAME='test-server', + SEGMENT_WRITE_KEY='123', APPOPTICS_API_TOKEN='123') def test_submission_tracking(self): """Test that submissions are tracked""" @@ -110,6 +111,13 @@ def test_submission_tracking(self): data = {'xml_submission_file': sf, 'media_file': f} request = self.factory.post(request_path, data) request.user = AnonymousUser() + request.META['HTTP_DATE'] = '2020-09-10T11:56:32.424726+00:00' + request.META['HTTP_REFERER'] = settings.HOSTNAME +\ + f':8000' + request.META['HTTP_USER_AGENT'] =\ + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit'\ + '/537.36 (KHTML, like Gecko) Chrome'\ + '/83.0.4103.61 Safari/537.36' response = view(request, username=self.user.username) self.assertContains(response, 'Successful submission', status_code=201) @@ -128,35 +136,35 @@ def test_submission_tracking(self): 'Submission created', { 'xform_id': self.xform.pk, + 'project_id': self.xform.project.pk, 'organization': 'Bob Inc.', 'from': 'XML Submissions', 'label': f'form-{form_id}-owned-by-{username}', + 'event_source': 'Submission collected from Enketo', 'value': 1, 'event_by': 'anonymous' }, - { - 'source': 'test-server', - 'organization': 'Bob Inc.', - 'event_by': 'anonymous', - 'action_from': 'XML Submissions', - 'xform_id': self.xform.pk, - 'path': f'/{username}/submission', - 'url': f'http://testserver/{username}/submission', + {'page': { + 'path': '/bob/submission', + 'referrer': settings.HOSTNAME + f':8000', + 'url': 'http://testserver/bob/submission' + }, + 'active': True, 'ip': '127.0.0.1', - 'userId': self.user.id - }) + 'userId': self.xform.user.pk, + 'receivedAt': '2020-09-10T11:56:32.424726+00:00', + 'userAgent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit' + '/537.36 (KHTML, like Gecko) Chrome' + '/83.0.4103.61 Safari/537.36'} + ) appoptics_mock.submit_measurement.assert_called_with( 'Submission created', 1, tags={ - 'source': 'test-server', - 'event_by': 'anonymous', - 'organization': 'Bob_Inc.', - 'action_from': 'XML_Submissions', - 'xform_id': self.xform.pk, - 'path': f'/{username}/submission', - 'url': f'http://testserver/{username}/submission', + 'active': True, 'ip': '127.0.0.1', - 'userId': self.user.id - }) + 'userId': self.xform.user.pk, + 'receivedAt': '2020-09-10T11:56:32.424726+00:00', + 'userAgent': 'Mozilla/5.0_X11;_Linux_x86_64_AppleWebKit' + '/537.36_KHTML,_like_Gecko_Chrome/83.0.4103.61_Safari/537.36'}) diff --git a/onadata/libs/utils/analytics.py b/onadata/libs/utils/analytics.py index 0812b10af0..8cd4eb344e 100644 --- a/onadata/libs/utils/analytics.py +++ b/onadata/libs/utils/analytics.py @@ -143,12 +143,33 @@ def get_event_label(self) -> str: event_label = f"form-{form_id}-owned-by-{username}" return event_label + def get_request_origin(self, request, tracking_properties): + if isinstance(self.tracked_obj, Instance): + try: + user_agent = request.META['HTTP_USER_AGENT'] + if 'Android' in user_agent: + event_source = f'Submission collected from ODK COLLECT' + elif 'Chrome' or 'Mozilla' or 'Safari' in user_agent: + event_source = f'Submission collected from Enketo' + except KeyError: + event_source = "" + else: + event_source = "" + tracking_properties.update({'event_source': event_source}) + return tracking_properties + def _track_object_event(self, obj, request=None) -> None: self.tracked_obj = obj self.set_user() event_name = self.get_event_name() label = self.get_event_label() tracking_properties = self.get_tracking_properties(label=label) + if tracking_properties['from'] == 'XML Submissions': + # Only introduce an `event_source` field for submissions + # created xml. This helps differentiate Enketo and ODK Collect + # instances. Otherwise, use the `from` tag in properties object + tracking_properties = self.get_request_origin( + request, tracking_properties) track( self.user, event_name, properties=tracking_properties, request=request) @@ -172,12 +193,11 @@ def decorator(obj, *args): def track(user, event_name, properties=None, context=None, request=None): """Record actions with the track() API call to the analytics agents.""" if _segment or appoptics_api: - context = context or {} - context['source'] = settings.HOSTNAME - - properties = properties or {} - user_id = get_user_id(user) + properties = properties or {} + context = context or {} + # Introduce inner page object within the context + context['page'] = {} if 'value' not in properties: properties['value'] = 1 @@ -186,25 +206,17 @@ def track(user, event_name, properties=None, context=None, request=None): submitted_by_user = properties.pop('submitted_by') submitted_by = get_user_id(submitted_by_user) properties['event_by'] = submitted_by - if submitted_by_user: - context['event_by'] = submitted_by_user.username - else: - context['event_by'] = submitted_by - - if 'organization' in properties: - context['organization'] = properties.get('organization') - - if 'from' in properties: - context['action_from'] = properties.get('from') - if 'xform_id' in properties: - context['xform_id'] = properties['xform_id'] + context['active'] = True if request: - context['path'] = request.path - context['url'] = request.build_absolute_uri() context['ip'] = request.META.get('REMOTE_ADDR', '') context['userId'] = user.id + context['receivedAt'] = request.META['HTTP_DATE'] + context['userAgent'] = request.META['HTTP_USER_AGENT'] + context['page']['path'] = request.path + context['page']['referrer'] = request.META['HTTP_REFERER'] + context['page']['url'] = request.build_absolute_uri() if _segment: segment_analytics.track(user_id, event_name, properties, context) diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 97980e71ae..86f714f7f2 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -466,6 +466,7 @@ def submit_csv(username, xform, csv_file, overwrite=False): event_name = None tracking_properties = { 'xform_id': xform.pk, + 'project_id': xform.project.pk, 'submitted_by': event_by, 'label': f'csv-import-for-form-{xform.pk}', 'from': 'CSV Import', From beb9cb8d060a6439eda34eb69c3c41bfe372d6b7 Mon Sep 17 00:00:00 2001 From: WinnyTroy Date: Thu, 1 Oct 2020 14:06:11 +0300 Subject: [PATCH 2/3] Track newly created Projects, Forms and User Accounts --- .../tests/viewsets/test_abstract_viewset.py | 14 +++++++++ .../libs/serializers/project_serializer.py | 8 +++++ .../serializers/user_profile_serializer.py | 6 ++++ onadata/libs/utils/analytics.py | 29 +++++++++++++------ onadata/libs/utils/common_tags.py | 3 ++ onadata/libs/utils/logger_tools.py | 17 +++++++++++ 6 files changed, 68 insertions(+), 9 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index 3e44532134..99f5b07aad 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -192,6 +192,13 @@ def _project_create(self, project_data={}, merge=True): request = self.factory.post( '/', data=json.dumps(data), content_type="application/json", **self.extra) + request.META['HTTP_DATE'] = '2020-09-10T11:56:32.424726+00:00' + request.META['HTTP_REFERER'] = settings.HOSTNAME +\ + f':8000' + request.META['HTTP_USER_AGENT'] =\ + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit'\ + '/537.36 (KHTML, like Gecko) Chrome'\ + '/83.0.4103.61 Safari/537.36' response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) self.project = Project.objects.filter( @@ -244,6 +251,13 @@ def _publish_xls_form_to_project(self, publish_data={}, merge=True, post_data = {'xls_file': xls_file} request = self.factory.post( '/', data=post_data, **self.extra) + request.META['HTTP_DATE'] = '2020-09-10T11:56:32.424726+00:00' + request.META['HTTP_REFERER'] = settings.HOSTNAME +\ + f':8000' + request.META['HTTP_USER_AGENT'] =\ + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit'\ + '/537.36 (KHTML, like Gecko) Chrome'\ + '/83.0.4103.61 Safari/537.36' response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) self.xform = XForm.objects.all().order_by('pk').reverse()[0] diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index da8d35a4c1..d9b00c357d 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -22,6 +22,7 @@ DataViewMinimalSerializer from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.tag_list_serializer import TagListSerializer +from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.cache_tools import ( PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, PROJ_NUM_DATASET_CACHE, PROJ_PERM_CACHE, PROJ_SUB_DATE_CACHE, PROJ_TEAM_USERS_CACHE, @@ -438,6 +439,13 @@ def update(self, instance, validated_data): return instance + @track_object_event( + user_field='created_by', + properties={ + 'created_by': 'created_by', + 'project_id': 'pk', + 'project_name': 'name'} + ) def create(self, validated_data): metadata = validated_data.get('metadata', dict()) if metadata is None: diff --git a/onadata/libs/serializers/user_profile_serializer.py b/onadata/libs/serializers/user_profile_serializer.py index 5dcce2d877..f2686c4131 100644 --- a/onadata/libs/serializers/user_profile_serializer.py +++ b/onadata/libs/serializers/user_profile_serializer.py @@ -29,6 +29,7 @@ from onadata.libs.permissions import CAN_VIEW_PROFILE, is_organization from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.utils.cache_tools import IS_ORG +from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.email import ( get_verification_url, get_verification_email_data ) @@ -229,6 +230,11 @@ def update(self, instance, validated_data): return super(UserProfileSerializer, self).update(instance, params) + @track_object_event( + user_field='user', + properties={ + 'name': 'name', + 'country': 'country'}) def create(self, validated_data): params = validated_data request = self.context.get('request') diff --git a/onadata/libs/utils/analytics.py b/onadata/libs/utils/analytics.py index 8cd4eb344e..75b5264989 100644 --- a/onadata/libs/utils/analytics.py +++ b/onadata/libs/utils/analytics.py @@ -12,9 +12,14 @@ from django.contrib.auth.models import User from django.utils import timezone -from onadata.apps.logger.models import Instance +from onadata.apps.logger.models import Instance, XForm, Project +from onadata.apps.main.models import UserProfile from onadata.libs.utils.common_tags import ( - INSTANCE_CREATE_EVENT, INSTANCE_UPDATE_EVENT) + INSTANCE_CREATE_EVENT, + INSTANCE_UPDATE_EVENT, + XFORM_CREATION_EVENT, + PROJECT_CREATION_EVENT, + USER_CREATION_EVENT) from onadata.libs.utils.common_tools import report_exception @@ -133,6 +138,12 @@ def get_event_name(self) -> str: event_name = INSTANCE_UPDATE_EVENT else: event_name = INSTANCE_CREATE_EVENT + elif isinstance(self.tracked_obj, XForm) and not event_name: + event_name = XFORM_CREATION_EVENT + elif isinstance(self.tracked_obj, Project): + event_name = PROJECT_CREATION_EVENT + elif isinstance(self.tracked_obj, UserProfile): + event_name = USER_CREATION_EVENT return event_name def get_event_label(self) -> str: @@ -155,7 +166,7 @@ def get_request_origin(self, request, tracking_properties): event_source = "" else: event_source = "" - tracking_properties.update({'event_source': event_source}) + tracking_properties.update({'from': event_source}) return tracking_properties def _track_object_event(self, obj, request=None) -> None: @@ -164,12 +175,12 @@ def _track_object_event(self, obj, request=None) -> None: event_name = self.get_event_name() label = self.get_event_label() tracking_properties = self.get_tracking_properties(label=label) - if tracking_properties['from'] == 'XML Submissions': - # Only introduce an `event_source` field for submissions - # created xml. This helps differentiate Enketo and ODK Collect - # instances. Otherwise, use the `from` tag in properties object - tracking_properties = self.get_request_origin( - request, tracking_properties) + try: + if tracking_properties['from'] == 'XML Submissions': + tracking_properties = self.get_request_origin( + request, tracking_properties) + except KeyError: + pass track( self.user, event_name, properties=tracking_properties, request=request) diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index 69dd7e87e5..775c0a1ee4 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -185,3 +185,6 @@ INSTANCE_CREATE_EVENT = 'Submission created' INSTANCE_UPDATE_EVENT = 'Submission updated' +XFORM_CREATION_EVENT = 'XForm created' +PROJECT_CREATION_EVENT = 'Project created' +USER_CREATION_EVENT = 'User account created' diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 7403892d1e..1557e53f85 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -48,6 +48,7 @@ from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.apps.viewer.models.parsed_instance import ParsedInstance from onadata.apps.viewer.signals import process_submission +from onadata.libs.utils.analytics import track_object_event from onadata.libs.utils.common_tags import METADATA_FIELDS from onadata.libs.utils.common_tools import report_exception, get_uuid from onadata.libs.utils.model_tools import set_uuid @@ -634,6 +635,14 @@ def publish_form(callback): return {'type': 'alert-error', 'text': text(e)} +@track_object_event( + user_field='user', + properties={ + 'created_by': 'user', + 'xform_id': 'pk', + 'xform_name': 'title'}, + additional_context={'from': 'Publish XLS Form'} +) @transaction.atomic() def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): """Create or update DataDictionary with xls_file, user @@ -655,6 +664,14 @@ def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): project=project) +@track_object_event( + user_field='user', + properties={ + 'created_by': 'user', + 'xform_id': 'pk', + 'xform_name': 'title'}, + additional_context={'from': 'Publish XML Form'} +) def publish_xml_form(xml_file, user, project, id_string=None, created_by=None): xml = xml_file.read() if isinstance(xml, bytes): From 1a9ea7fef2fe9480f6d66adace0668f82f510569 Mon Sep 17 00:00:00 2001 From: WinnyTroy Date: Wed, 11 Nov 2020 14:48:16 +0300 Subject: [PATCH 3/3] Remove AppOptics calls. Stems from here https://onaio.slack.com/archives/C0402U49R/p1599567686026500 --- .../tests/viewsets/test_abstract_viewset.py | 14 ---- onadata/libs/tests/utils/test_analytics.py | 74 ++++++------------- onadata/libs/utils/analytics.py | 63 +++------------- 3 files changed, 34 insertions(+), 117 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index 99f5b07aad..3e44532134 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -192,13 +192,6 @@ def _project_create(self, project_data={}, merge=True): request = self.factory.post( '/', data=json.dumps(data), content_type="application/json", **self.extra) - request.META['HTTP_DATE'] = '2020-09-10T11:56:32.424726+00:00' - request.META['HTTP_REFERER'] = settings.HOSTNAME +\ - f':8000' - request.META['HTTP_USER_AGENT'] =\ - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit'\ - '/537.36 (KHTML, like Gecko) Chrome'\ - '/83.0.4103.61 Safari/537.36' response = view(request, owner=self.user.username) self.assertEqual(response.status_code, 201) self.project = Project.objects.filter( @@ -251,13 +244,6 @@ def _publish_xls_form_to_project(self, publish_data={}, merge=True, post_data = {'xls_file': xls_file} request = self.factory.post( '/', data=post_data, **self.extra) - request.META['HTTP_DATE'] = '2020-09-10T11:56:32.424726+00:00' - request.META['HTTP_REFERER'] = settings.HOSTNAME +\ - f':8000' - request.META['HTTP_USER_AGENT'] =\ - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit'\ - '/537.36 (KHTML, like Gecko) Chrome'\ - '/83.0.4103.61 Safari/537.36' response = view(request, pk=project_id) self.assertEqual(response.status_code, 201) self.xform = XForm.objects.all().order_by('pk').reverse()[0] diff --git a/onadata/libs/tests/utils/test_analytics.py b/onadata/libs/tests/utils/test_analytics.py index 137045d06e..930574a959 100644 --- a/onadata/libs/tests/utils/test_analytics.py +++ b/onadata/libs/tests/utils/test_analytics.py @@ -15,7 +15,7 @@ TestAbstractViewSet from onadata.apps.api.viewsets.xform_submission_viewset import \ XFormSubmissionViewSet -from onadata.libs.utils.analytics import get_user_id, sanitize_metric_values +from onadata.libs.utils.analytics import get_user_id class TestAnalytics(TestAbstractViewSet): @@ -47,55 +47,39 @@ def test_track(self): user1.username, 'testing track function', {'value': 1}, - {'page': {}, 'active': True}) - - def test_sanitize_metric_values(self): - """Test sanitize_metric_values()""" - expected_output = { - 'action_from': 'XML_Submissions', - 'event_by': 'bob', - 'ip': '18.203.134.101', - 'path': '/enketo/1814/submission', - 'source': 'onadata-ona-stage', - 'url': 'https://stage-api.ona.io/enketo/1814/submission', - 'userId': 1, - 'xform_id': 1} - context = { - 'action_from': 'XML Submissions', - 'event_by': 'bob', - 'ip': '18.203.134.101', - 'library': { - 'name': 'analytics-python', - 'version': '1.2.9' - }, - 'organization': '', - 'path': '/enketo/1814/submission', - 'source': 'onadata-ona-stage', - 'url': 'https://stage-api.ona.io/enketo/1814/submission', - 'userId': 1, - 'xform_id': 1} - - self.assertEqual(sanitize_metric_values(context), expected_output) + {'page': {}, 'campaign': {}, 'active': True}) @override_settings( - SEGMENT_WRITE_KEY='123', - APPOPTICS_API_TOKEN='123') + SEGMENT_WRITE_KEY='123') def test_submission_tracking(self): """Test that submissions are tracked""" segment_mock = MagicMock() - appoptics_mock = MagicMock() onadata.libs.utils.analytics.segment_analytics = segment_mock onadata.libs.utils.analytics.init_analytics() self.assertEqual(segment_mock.write_key, '123') # Test out that the track_object_event decorator - # Tracks created submissions + # Tracks created submissions, XForms and Projects view = XFormSubmissionViewSet.as_view({ 'post': 'create', 'head': 'create' }) self._publish_xls_form_to_project() - onadata.libs.utils.analytics.appoptics_api = appoptics_mock + segment_mock.track.assert_called_with( + 'bob@columbia.edu', + 'XForm created', + { + 'created_by': self.xform.user, + 'xform_id': self.xform.pk, + 'xform_name': self.xform.title, + 'from': 'Publish XLS Form', + 'value': 1 + }, + { + 'page': {}, + 'campaign': {}, + 'active': True + }) s = self.surveys[0] media_file = "1335783522563.jpg" path = os.path.join(self.main_directory, 'fixtures', @@ -113,7 +97,7 @@ def test_submission_tracking(self): request.user = AnonymousUser() request.META['HTTP_DATE'] = '2020-09-10T11:56:32.424726+00:00' request.META['HTTP_REFERER'] = settings.HOSTNAME +\ - f':8000' + ':8000' request.META['HTTP_USER_AGENT'] =\ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit'\ '/537.36 (KHTML, like Gecko) Chrome'\ @@ -138,17 +122,18 @@ def test_submission_tracking(self): 'xform_id': self.xform.pk, 'project_id': self.xform.project.pk, 'organization': 'Bob Inc.', - 'from': 'XML Submissions', + 'from': 'Submission collected from Enketo', 'label': f'form-{form_id}-owned-by-{username}', - 'event_source': 'Submission collected from Enketo', 'value': 1, 'event_by': 'anonymous' }, {'page': { 'path': '/bob/submission', - 'referrer': settings.HOSTNAME + f':8000', + 'referrer': settings.HOSTNAME + ':8000', 'url': 'http://testserver/bob/submission' }, + 'campaign': { + 'source': settings.HOSTNAME}, 'active': True, 'ip': '127.0.0.1', 'userId': self.xform.user.pk, @@ -157,14 +142,3 @@ def test_submission_tracking(self): '/537.36 (KHTML, like Gecko) Chrome' '/83.0.4103.61 Safari/537.36'} ) - - appoptics_mock.submit_measurement.assert_called_with( - 'Submission created', - 1, - tags={ - 'active': True, - 'ip': '127.0.0.1', - 'userId': self.xform.user.pk, - 'receivedAt': '2020-09-10T11:56:32.424726+00:00', - 'userAgent': 'Mozilla/5.0_X11;_Linux_x86_64_AppleWebKit' - '/537.36_KHTML,_like_Gecko_Chrome/83.0.4103.61_Safari/537.36'}) diff --git a/onadata/libs/utils/analytics.py b/onadata/libs/utils/analytics.py index 75b5264989..4a29c6c338 100644 --- a/onadata/libs/utils/analytics.py +++ b/onadata/libs/utils/analytics.py @@ -1,12 +1,8 @@ # -*- coding: utf-8 -*- # Analytics package for tracking and measuring with services like Segment. # Heavily borrowed from RapidPro's temba.utils.analytics -import sys - import analytics as segment_analytics -from typing import Dict, List, Optional, Any -import appoptics_metrics -from appoptics_metrics import sanitize_metric_name, exceptions +from typing import Dict, List, Optional from django.conf import settings from django.contrib.auth.models import User @@ -20,49 +16,18 @@ XFORM_CREATION_EVENT, PROJECT_CREATION_EVENT, USER_CREATION_EVENT) -from onadata.libs.utils.common_tools import report_exception - -appoptics_api = None _segment = False def init_analytics(): """Initialize the analytics agents with write credentials.""" segment_write_key = getattr(settings, 'SEGMENT_WRITE_KEY', None) - appoptics_api_token = getattr(settings, "APPOPTICS_API_TOKEN", None) if segment_write_key: global _segment segment_analytics.write_key = segment_write_key _segment = True - if appoptics_api_token: - global appoptics_api - appoptics_api = appoptics_metrics.connect( - appoptics_api_token, sanitizer=sanitize_metric_name) - - -def sanitize_metric_values(data: Dict[str, Any]) -> Dict[str, Any]: - """ - Sanitizes values in order to ensure the values are valid - for AppOptics - """ - sanitized_data = data.copy() - for key, value in data.items(): - new_value = value - if new_value and not isinstance(new_value, dict): - if isinstance(new_value, str): - if ' ' in new_value: - new_value = new_value.replace(' ', '_') - - if '(' in value or ')' in new_value: - new_value = new_value.replace(')', "").replace('(', "") - - sanitized_data.update({key: new_value}) - else: - sanitized_data.pop(key) - return sanitized_data - def get_user_id(user): """Return a user email or username or the string 'anonymous'""" @@ -159,9 +124,9 @@ def get_request_origin(self, request, tracking_properties): try: user_agent = request.META['HTTP_USER_AGENT'] if 'Android' in user_agent: - event_source = f'Submission collected from ODK COLLECT' + event_source = 'Submission collected from ODK COLLECT' elif 'Chrome' or 'Mozilla' or 'Safari' in user_agent: - event_source = f'Submission collected from Enketo' + event_source = 'Submission collected from Enketo' except KeyError: event_source = "" else: @@ -203,12 +168,13 @@ def decorator(obj, *args): def track(user, event_name, properties=None, context=None, request=None): """Record actions with the track() API call to the analytics agents.""" - if _segment or appoptics_api: + if _segment: user_id = get_user_id(user) properties = properties or {} context = context or {} - # Introduce inner page object within the context + # Introduce inner page and campaign object within the context context['page'] = {} + context['campaign'] = {} if 'value' not in properties: properties['value'] = 1 @@ -223,21 +189,12 @@ def track(user, event_name, properties=None, context=None, request=None): if request: context['ip'] = request.META.get('REMOTE_ADDR', '') context['userId'] = user.id - context['receivedAt'] = request.META['HTTP_DATE'] - context['userAgent'] = request.META['HTTP_USER_AGENT'] + context['receivedAt'] = request.META.get('HTTP_DATE', '') + context['userAgent'] = request.META.get('HTTP_USER_AGENT', '') + context['campaign']['source'] = settings.HOSTNAME context['page']['path'] = request.path - context['page']['referrer'] = request.META['HTTP_REFERER'] + context['page']['referrer'] = request.META.get('HTTP_REFERER', '') context['page']['url'] = request.build_absolute_uri() if _segment: segment_analytics.track(user_id, event_name, properties, context) - - if appoptics_api: - tags = sanitize_metric_values(context) - try: - appoptics_api.submit_measurement( - event_name, - properties['value'], - tags=tags) - except exceptions.BadRequest as e: - report_exception("Bad AppOptics Request", e, sys.exc_info())