From f0660becb18f97d44813bb518de7dfcce08d91f0 Mon Sep 17 00:00:00 2001 From: WinnyTroy Date: Tue, 12 Jan 2021 12:00:55 +0300 Subject: [PATCH 1/2] Update Tableau Documentation --- docs/index.rst | 8 + docs/onadata-tableau.rst | 270 ++++++++++++++++++ .../tests/viewsets/test_tableau_viewset.py | 21 +- 3 files changed, 297 insertions(+), 2 deletions(-) create mode 100644 docs/onadata-tableau.rst diff --git a/docs/index.rst b/docs/index.rst index 30eaa50de5..1ed454510e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,6 +73,14 @@ Flow Results Packages flow-results +Onadata-Tableau Intergration +------------------------------ + +.. toctree:: + :maxdepth: 2 + + onadata-tableau + Ona Tagging API ~~~~~~~~~~~~~~~ diff --git a/docs/onadata-tableau.rst b/docs/onadata-tableau.rst new file mode 100644 index 0000000000..ac7cac300c --- /dev/null +++ b/docs/onadata-tableau.rst @@ -0,0 +1,270 @@ +Onadata-Tableau +*************** + +Visualize data collected with the onadata application on |Tableau|. This endpoint provides access to submitted data being pushed to Tableau via the Web Data Connector in JSON format. + +.. |Tableau| raw:: html + + Tableau + + +Where: + +- ``uuid`` - the form open data unique identifier + + +Tableau Web Data Connector Endpoints +------------------------------------ + +Schema Endpoint Example +^^^^^^^^^^^^^^^^^^^^^^^ +:: + + curl -X GET /api/v1/open-data-v2/24fde84caec342a19a7f2e3ea0c36e3f/schema + +Response +^^^^^^^^ +:: + + [ + { + "table_alias": "data", + "connection_name": "22_test", + "column_headers": [ + { + "id": "_id", + "dataType": "int", + "alias": "_id" + }, + { + "id": "country", + "dataType": "string", + "alias": "country" + }, + { + "id": "note", + "dataType": "string", + "alias": "note" + }, + { + "id": "user_select", + "dataType": "string", + "alias": "user_select" + }, + { + "id": "photo", + "dataType": "string", + "alias": "photo" + }, + { + "id": "meta_instanceID", + "dataType": "string", + "alias": "meta_instanceID" + } + ] + } + ] + +Forms with nested repeats will generate multiple table schemas for Tableau + +Response +^^^^^^^^ +:: + + [ + { + "table_alias": "data", + "connection_name": "22_transportation_new_form", + "column_headers": [ + { + "id": "_id", + "dataType": "int", + "alias": "_id" + }, + { + "id": "hospital_name", + "dataType": "string", + "alias": "hospital_name" + }, + { + "id": "hospital_hiv_medication_food_cake", + "dataType": "string", + "alias": "hospital_hiv_medication_food_cake" + }, + { + "id": "hospital_hiv_medication_food_cheese", + "dataType": "string", + "alias": "hospital_hiv_medication_food_cheese" + }, + { + "id": "hospital_hiv_medication_food_ham", + "dataType": "string", + "alias": "hospital_hiv_medication_food_ham" + }, + { + "id": "hospital_hiv_medication_food_vegetables", + "dataType": "string", + "alias": "hospital_hiv_medication_food_vegetables" + }, + { + "id": "hospital_hiv_medication_have_hiv_medication", + "dataType": "string", + "alias": "hospital_hiv_medication_have_hiv_medication" + }, + { + "id": "hospital_hiv_medication__gps_latitude", + "dataType": "string", + "alias": "hospital_hiv_medication__gps_latitude" + }, + { + "id": "hospital_hiv_medication__gps_longitude", + "dataType": "string", + "alias": "hospital_hiv_medication__gps_longitude" + }, + { + "id": "hospital_hiv_medication__gps_altitude", + "dataType": "string", + "alias": "hospital_hiv_medication__gps_altitude" + }, + { + "id": "hospital_hiv_medication__gps_precision", + "dataType": "string", + "alias": "hospital_hiv_medication__gps_precision" + }, + { + "id": "meta_instanceID", + "dataType": "string", + "alias": "meta_instanceID" + } + ] + }, + { + "table_alias": "person_repeat", + "connection_name": "22_transportation_new_form_person_repeat", + "column_headers": [ + { + "id": "_id", + "dataType": "int", + "alias": "_id" + }, + { + "id": "__parent_id", + "dataType": "int", + "alias": "__parent_id" + }, + { + "id": "__parent_table", + "dataType": "string", + "alias": "__parent_table" + }, + { + "id": "hospital_hiv_medication_person_first_name", + "dataType": "string", + "alias": "hospital_hiv_medication_person_first_name" + }, + { + "id": "hospital_hiv_medication_person_last_name", + "dataType": "string", + "alias": "hospital_hiv_medication_person_last_name" + }, + { + "id": "hospital_hiv_medication_person_food_cake", + "dataType": "string", + "alias": "hospital_hiv_medication_person_food_cake" + }, + { + "id": "hospital_hiv_medication_person_food_cheese", + "dataType": "string", + "alias": "hospital_hiv_medication_person_food_cheese" + }, + { + "id": "hospital_hiv_medication_person_food_ham", + "dataType": "string", + "alias": "hospital_hiv_medication_person_food_ham" + }, + { + "id": "hospital_hiv_medication_person_food_vegetables", + "dataType": "string", + "alias": "hospital_hiv_medication_person_food_vegetables" + }, + { + "id": "hospital_hiv_medication_person_have_hiv_medication", + "dataType": "string", + "alias": "hospital_hiv_medication_person_have_hiv_medication" + }, + { + "id": "hospital_hiv_medication_person_age", + "dataType": "int", + "alias": "hospital_hiv_medication_person_age" + }, + { + "id": "hospital_hiv_medication_person__gps_latitude", + "dataType": "string", + "alias": "hospital_hiv_medication_person__gps_latitude" + }, + { + "id": "hospital_hiv_medication_person__gps_longitude", + "dataType": "string", + "alias": "hospital_hiv_medication_person__gps_longitude" + }, + { + "id": "hospital_hiv_medication_person__gps_altitude", + "dataType": "string", + "alias": "hospital_hiv_medication_person__gps_altitude" + }, + { + "id": "hospital_hiv_medication_person__gps_precision", + "dataType": "string", + "alias": "hospital_hiv_medication_person__gps_precision" + } + ] + } + ] + + +Data Endpoint Example +^^^^^^^^^^^^^^^^^^^^^ +:: + + curl -X GET /api/v1/open-data-v2/5d3da685cbe64fc6b97a1b03ffccd847/data + +Response +^^^^^^^^ +:: + + [ + { + "_id": 4, + "hospital_name": "Melkizedek", + "meta_instanceID": "uuid:f0be8145-b840-4fde-a531-a38aeb1260f4", + "hospital_hiv_medication__gps_latitude": "-1.302025", + "hospital_hiv_medication__gps_longitude": "36.745877", + "hospital_hiv_medication__gps_altitude": "0", + "hospital_hiv_medication__gps_precision": "0", + "hospital_hiv_medication_food_cake": "TRUE", + "hospital_hiv_medication_food_cheese": "TRUE", + "hospital_hiv_medication_food_ham": "TRUE", + "hospital_hiv_medication_food_vegetables": "TRUE", + "person_repeat": [ + { + "__parent_id": 4, + "__parent_table": "data", + "_id": 16, + "hospital_hiv_medication_person_age": 43, + "hospital_hiv_medication_person__gps_latitude": "-1.302819", + "hospital_hiv_medication_person__gps_longitude": "36.746857", + "hospital_hiv_medication_person__gps_altitude": "0", + "hospital_hiv_medication_person__gps_precision": "0", + "hospital_hiv_medication_person_food_cake": "TRUE", + "hospital_hiv_medication_person_food_cheese": "TRUE", + "hospital_hiv_medication_person_food_ham": "TRUE", + "hospital_hiv_medication_person_food_vegetables": "TRUE", + "hospital_hiv_medication_person_last_name": "Kendrik", + "hospital_hiv_medication_person_first_name": "Tom", + "hospital_hiv_medication_person_have_hiv_medication": "no" + } + ], + "hospital_hiv_medication_have_hiv_medication": "yes" + } + ] diff --git a/onadata/apps/api/tests/viewsets/test_tableau_viewset.py b/onadata/apps/api/tests/viewsets/test_tableau_viewset.py index 16a95a6b02..73ed3eb316 100644 --- a/onadata/apps/api/tests/viewsets/test_tableau_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_tableau_viewset.py @@ -4,14 +4,14 @@ """ import os import json - +from re import search from django.test import RequestFactory from django.utils.dateparse import parse_datetime from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models.open_data import get_or_create_opendata from onadata.apps.api.viewsets.tableau_viewset import ( TableauViewSet, unpack_select_multiple_data, - unpack_gps_data) + unpack_gps_data, clean_xform_headers) from onadata.libs.renderers.renderers import pairing @@ -299,3 +299,20 @@ def test_unpack_gps_data(self): '_gps_precision': '0' } self.assertEqual(data, expected_data) + + def test_clean_xform_headers(self): + """ + Test that column header fields for group columns + do not contain indexing when schema columns + are being pushed to Tableau. + """ + headers = self.xform.get_headers(repeat_iterations=1) + group_columns = [ + field for field in headers if search(r"\[+\d+\]", field)] + self.assertEqual(group_columns, + ['children[1]/childs_name', + 'children[1]/childs_age']) + + cleaned_data = clean_xform_headers(group_columns) + self.assertEqual(cleaned_data, + ['childs_name', 'childs_age']) From 190de6cb066813cc230f1381fdb5d24a64afbf98 Mon Sep 17 00:00:00 2001 From: WinnyTroy Date: Fri, 15 Jan 2021 17:43:48 +0300 Subject: [PATCH 2/2] Introduce `api/v2` endpoints. Migrate new viewset to fall under v2 versioning endpoints --- docs/onadata-tableau.rst | 4 ++-- .../apps/api/tests/viewsets/test_ona_api.py | 24 +++++++++++++++---- .../tests/viewsets/test_tableau_viewset.py | 2 +- onadata/apps/api/urls/__init__.py | 0 onadata/apps/api/{urls.py => urls/v1_urls.py} | 2 -- onadata/apps/api/urls/v2_urls.py | 9 +++++++ onadata/apps/api/viewsets/v2/__init__.py | 0 .../api/viewsets/{ => v2}/tableau_viewset.py | 0 onadata/apps/main/urls.py | 12 ++++++---- 9 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 onadata/apps/api/urls/__init__.py rename onadata/apps/api/{urls.py => urls/v1_urls.py} (98%) create mode 100644 onadata/apps/api/urls/v2_urls.py create mode 100644 onadata/apps/api/viewsets/v2/__init__.py rename onadata/apps/api/viewsets/{ => v2}/tableau_viewset.py (100%) diff --git a/docs/onadata-tableau.rst b/docs/onadata-tableau.rst index ac7cac300c..a48143b685 100644 --- a/docs/onadata-tableau.rst +++ b/docs/onadata-tableau.rst @@ -21,7 +21,7 @@ Schema Endpoint Example ^^^^^^^^^^^^^^^^^^^^^^^ :: - curl -X GET /api/v1/open-data-v2/24fde84caec342a19a7f2e3ea0c36e3f/schema + curl -X GET /api/v2/open-data/24fde84caec342a19a7f2e3ea0c36e3f/schema Response ^^^^^^^^ @@ -227,7 +227,7 @@ Data Endpoint Example ^^^^^^^^^^^^^^^^^^^^^ :: - curl -X GET /api/v1/open-data-v2/5d3da685cbe64fc6b97a1b03ffccd847/data + curl -X GET /api/v2/open-data/5d3da685cbe64fc6b97a1b03ffccd847/data Response ^^^^^^^^ diff --git a/onadata/apps/api/tests/viewsets/test_ona_api.py b/onadata/apps/api/tests/viewsets/test_ona_api.py index 625932ac0f..98a64f40ab 100644 --- a/onadata/apps/api/tests/viewsets/test_ona_api.py +++ b/onadata/apps/api/tests/viewsets/test_ona_api.py @@ -2,18 +2,32 @@ from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ TestAbstractViewSet -from onadata.apps.api.urls import router +from onadata.apps.api.urls.v1_urls import router as v1_router +from onadata.apps.api.urls.v2_urls import router as v2_router class TestOnaApi(TestAbstractViewSet): - def test_number_of_viewsets(self): + def test_number_of_v1_viewsets(self): ''' - Counts the number of viewsets + Counts the number of v1 viewsets + for the api django app ''' - view = router.get_api_root_view() + view = v1_router.get_api_root_view() path = '/api/v1/' request = self.factory.get(path) request.resolver_match = resolve(path) response = view(request) - self.assertEquals(len(response.data), 30) + self.assertEquals(len(response.data), 29) + + def test_number_of_v2_viewsets(self): + ''' + Counts the number of v2 viewsets + for the api django app + ''' + view = v2_router.get_api_root_view() + path = '/api/v2/' + request = self.factory.get(path) + request.resolver_match = resolve(path) + response = view(request) + self.assertEquals(len(response.data), 1) diff --git a/onadata/apps/api/tests/viewsets/test_tableau_viewset.py b/onadata/apps/api/tests/viewsets/test_tableau_viewset.py index 73ed3eb316..009e5ba45f 100644 --- a/onadata/apps/api/tests/viewsets/test_tableau_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_tableau_viewset.py @@ -9,7 +9,7 @@ from django.utils.dateparse import parse_datetime from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models.open_data import get_or_create_opendata -from onadata.apps.api.viewsets.tableau_viewset import ( +from onadata.apps.api.viewsets.v2.tableau_viewset import ( TableauViewSet, unpack_select_multiple_data, unpack_gps_data, clean_xform_headers) from onadata.libs.renderers.renderers import pairing diff --git a/onadata/apps/api/urls/__init__.py b/onadata/apps/api/urls/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/onadata/apps/api/urls.py b/onadata/apps/api/urls/v1_urls.py similarity index 98% rename from onadata/apps/api/urls.py rename to onadata/apps/api/urls/v1_urls.py index 135952fb25..631fb96937 100644 --- a/onadata/apps/api/urls.py +++ b/onadata/apps/api/urls/v1_urls.py @@ -21,7 +21,6 @@ from onadata.apps.api.viewsets.metadata_viewset import MetaDataViewSet from onadata.apps.api.viewsets.note_viewset import NoteViewSet from onadata.apps.api.viewsets.open_data_viewset import OpenDataViewSet -from onadata.apps.api.viewsets.tableau_viewset import TableauViewSet from onadata.apps.api.viewsets.organization_profile_viewset import \ OrganizationProfileViewSet from onadata.apps.api.viewsets.osm_viewset import OsmViewSet @@ -134,7 +133,6 @@ def get_urls(self): router.register(r'metadata', MetaDataViewSet, basename='metadata') router.register(r'notes', NoteViewSet) router.register(r'open-data', OpenDataViewSet, basename='open-data') -router.register(r'open-data-v2', TableauViewSet, basename='open-data-v2') router.register(r'orgs', OrganizationProfileViewSet) router.register(r'osm', OsmViewSet, basename='osm') router.register( diff --git a/onadata/apps/api/urls/v2_urls.py b/onadata/apps/api/urls/v2_urls.py new file mode 100644 index 0000000000..34da69ad81 --- /dev/null +++ b/onadata/apps/api/urls/v2_urls.py @@ -0,0 +1,9 @@ +# -*- coding=utf-8 -*- +""" +Custom rest_framework Router V2 +""" +from .v1_urls import MultiLookupRouter +from onadata.apps.api.viewsets.v2.tableau_viewset import TableauViewSet + +router = MultiLookupRouter(trailing_slash=False) +router.register(r'open-data', TableauViewSet, basename='open-data') diff --git a/onadata/apps/api/viewsets/v2/__init__.py b/onadata/apps/api/viewsets/v2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/onadata/apps/api/viewsets/tableau_viewset.py b/onadata/apps/api/viewsets/v2/tableau_viewset.py similarity index 100% rename from onadata/apps/api/viewsets/tableau_viewset.py rename to onadata/apps/api/viewsets/v2/tableau_viewset.py diff --git a/onadata/apps/main/urls.py b/onadata/apps/main/urls.py index ec38d27c11..7ded7dc5f5 100644 --- a/onadata/apps/main/urls.py +++ b/onadata/apps/main/urls.py @@ -9,13 +9,14 @@ from django.views.generic import RedirectView from onadata.apps import sms_support -from onadata.apps.api.urls import router -from onadata.apps.api.urls import XFormListViewSet +from onadata.apps.api.urls.v1_urls import router as api_v1_router +from onadata.apps.api.urls.v2_urls import router as api_v2_router +from onadata.apps.api.urls.v1_urls import XFormListViewSet from onadata.apps.api.viewsets.xform_list_viewset import ( PreviewXFormListViewSet ) -from onadata.apps.api.urls import XFormSubmissionViewSet -from onadata.apps.api.urls import BriefcaseViewset +from onadata.apps.api.urls.v1_urls import XFormSubmissionViewSet +from onadata.apps.api.urls.v1_urls import BriefcaseViewset from onadata.apps.logger import views as logger_views from onadata.apps.main import views as main_views from onadata.apps.main.registration_urls import ( @@ -39,7 +40,8 @@ urlpatterns = [ # change Language re_path(r'^i18n/', include(i18n)), - url('^api/v1/', include(router.urls)), + url('^api/v1/', include(api_v1_router.urls)), + url('^api/v2/', include(api_v2_router.urls)), re_path(r'^api-docs/', RedirectView.as_view(url=settings.STATIC_DOC, permanent=True)), re_path(r'^api/$',