Return flow data results for merged datasets

onadata/apps/api/tests/viewsets/test_floip_viewset.py

@@ -4,12 +4,15 @@
 """
 import json
 import os
+import uuid as uu
 from builtins import open
 
 from onadata.apps.api.tests.viewsets.test_abstract_viewset import \
     TestAbstractViewSet
 from onadata.apps.api.viewsets.floip_viewset import FloipViewSet
+from onadata.apps.api.viewsets.merged_xform_viewset import MergedXFormViewSet
 from onadata.apps.logger.models import Instance, XForm
+from onadata.libs.utils.user_auth import get_user_default_project
 
 
 class TestFloipViewSet(TestAbstractViewSet):
@@ -106,6 +109,28 @@ def test_retrieve_package(self):
         rendered_data = json.loads(response.rendered_content)
         self.assertEqual(rendered_data['data']['id'], data['id'])
 
+        # Test able to retrieve package using a complete uuid4 string
+        data_id = uu.UUID(data['id'], version=4)
+        response = view(request, uuid=str(data_id))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data, data)
+
+        # Test able to retrieve package using only the hex
+        # characters of a uuid string
+        response = view(request, uuid=data_id.hex)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data, data)
+
+        # Test able to retrieve public package
+        form: XForm = XForm.objects.filter(uuid=data['id']).first()
+        form.shared = True
+        form.shared_data = True
+        form.save()
+        data['modified'] = form.date_modified
+        response = view(request, uuid=str(data_id))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data, data)
+
     def test_update_package(self):
         """
         Test updating a specific package.
@@ -226,3 +251,58 @@ def test_responses_endpoint_format(self):
             response.data['attributes']['responses'])
 
         self.assertEqual(response.data, correct_response_format['data'])
+        # The FLOIP Endpoint should always return the complete uuid
+        # hex digits + dashes
+        self.assertEqual(len(response.data['id']), 36)
+
+    # pylint:disable=invalid-name
+    def test_retrieve_responses_merged_dataset(self):
+        """
+        Test that a user is able to retrieve FLOIP Responses for Merged
+        XForms
+        """
+        MD = """
+            | survey  |
+            |         | type  | name   | label   |
+            |         | photo | image1 | Photo   |
+            """
+        # Create Merged XForm
+        merged_dataset_view = MergedXFormViewSet.as_view({
+            'post': 'create',
+        })
+
+        project = get_user_default_project(self.user)
+        self._publish_xls_form_to_project()
+        self._make_submissions()
+        xform = self._publish_markdown(MD, self.user, id_string='a')
+
+        data = {
+            'xforms': [
+                f'http://testserver/api/v1/forms/{self.xform.pk}',
+                f'http://testserver/api/v1/forms/{xform.pk}',
+            ],
+            'name':
+            'Merged Dataset',
+            'project':
+            f'http://testserver/api/v1/projects/{project.pk}',
+        }
+
+        request = self.factory.post('/', data=data, **self.extra)
+        response = merged_dataset_view(request)
+        self.assertEqual(response.status_code, 201)
+        dataset_uuid = response.data['uuid']
+
+        # Assert that it's possible to retrieve the responses
+        view = FloipViewSet.as_view({'get': 'responses'})
+        request = self.factory.get(
+            f'/flow-results/packages/{dataset_uuid}/responses',
+            content_type='application/vnd.api+json', **self.extra)
+        response = view(request, uuid=dataset_uuid)
+        self.assertEqual(response.status_code, 200)
+
+        # Convert the returned generator object into a list
+        response.data['attributes']['responses'] = list(
+            response.data['attributes']['responses'])
+        # The transportation form(self.xform) contains 11 responses
+        # Assert that the responses are returned
+        self.assertEqual(len(response.data['attributes']['responses']), 11)

onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py

@@ -129,6 +129,8 @@ def _create_merged_dataset(self, geo=False):
             'project_name': self.project.name
         }
         self.assertEqual(response.data['xforms'][0], expected_xforms_data)
+        self.assertIsNotNone(response.data['uuid'])
+        self.assertEqual(len(response.data['uuid']), 32)
 
         return response.data
 

onadata/apps/api/viewsets/floip_viewset.py

@@ -2,7 +2,10 @@
 """
 FloipViewSet: API endpoint for /api/floip
 """
+from uuid import UUID
 
+from django.db.models import Q
+from django.shortcuts import get_object_or_404
 from rest_framework import mixins, status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
@@ -12,7 +15,7 @@
 from rest_framework_json_api.renderers import JSONRenderer
 
 from onadata.apps.api.permissions import XFormPermissions
-from onadata.apps.logger.models import XForm
+from onadata.apps.logger.models import XForm, Instance
 from onadata.libs import filters
 from onadata.libs.renderers.renderers import floip_list
 from onadata.libs.serializers.floip_serializer import (
@@ -60,6 +63,17 @@ class FloipViewSet(mixins.CreateModelMixin, mixins.DestroyModelMixin,
 
     lookup_field = 'uuid'
 
+    def get_object(self):
+        queryset = self.filter_queryset(self.get_queryset())
+        uuid = self.kwargs.get(self.lookup_field)
+        uuid = UUID(uuid, version=4)
+        obj = get_object_or_404(queryset, Q(uuid=uuid.hex) | Q(uuid=str(uuid)))
+        self.check_object_permissions(self.request, obj)
+
+        if self.request.user.is_anonymous and obj.require_auth:
+            self.permission_denied(self.request)
+        return obj
+
     def get_serializer_class(self):
         if self.action == 'list':
             return FloipListSerializer
@@ -72,8 +86,9 @@ def get_serializer_class(self):
     def get_success_headers(self, data):
         headers = super(FloipViewSet, self).get_success_headers(data)
         headers['Content-Type'] = 'application/vnd.api+json'
+        uuid = str(UUID(data['id']))
         headers['Location'] = self.request.build_absolute_uri(
-            reverse('flow-results-detail', kwargs={'uuid': data['id']}))
+            reverse('flow-results-detail', kwargs={'uuid': uuid}))
 
         return headers
 
@@ -84,15 +99,16 @@ def responses(self, request, uuid=None):
         """
         status_code = status.HTTP_200_OK
         xform = self.get_object()
+        uuid = str(UUID(uuid or xform.uuid, version=4))
         data = {
-            "id": uuid or xform.uuid,
+            "id": uuid,
             "type": "flow-results-data",
             "attributes": {}
         }
         headers = {
             'Content-Type': 'application/vnd.api+json',
             'Location': self.request.build_absolute_uri(
-                reverse('flow-results-responses', kwargs={'uuid': xform.uuid}))
+                reverse('flow-results-responses', kwargs={'uuid': uuid}))
         }  # yapf: disable
         if request.method == 'POST':
             serializer = FlowResultsResponseSerializer(
@@ -105,7 +121,16 @@ def responses(self, request, uuid=None):
             else:
                 status_code = status.HTTP_201_CREATED
         else:
-            queryset = xform.instances.values_list('json', flat=True)
+            if xform.is_merged_dataset:
+                pks = xform.mergedxform.xforms.filter(
+                    deleted_at__isnull=True
+                ).values_list('pk', flat=True)
+                queryset = Instance.objects.filter(
+                    xform_id__in=pks,
+                    deleted_at__isnull=True).values_list('json', flat=True)
+            else:
+                queryset = xform.instances.values_list('json', flat=True)
+
             paginate_queryset = self.paginate_queryset(queryset)
             if paginate_queryset:
                 data['attributes']['responses'] = floip_list(paginate_queryset)

onadata/apps/logger/migrations/0061_auto_20200713_0814.py

@@ -0,0 +1,26 @@
+# pylint: skip-file
+# Generated by Django 2.2.10 on 2020-07-13 12:14
+
+from django.db import migrations
+from onadata.libs.utils.common_tools import get_uuid
+
+
+def generate_uuid_if_missing(apps, schema_editor):
+    """
+    Generate uuids for XForms without them
+    """
+    XForm = apps.get_model('logger', 'XForm')
+
+    for xform in XForm.objects.filter(uuid=''):
+        xform.uuid = get_uuid()
+        xform.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('logger', '0060_auto_20200305_0357'),
+    ]
+
+    operations = [
+        migrations.RunPython(generate_uuid_if_missing)]

onadata/apps/logger/models/merged_xform.py

@@ -2,6 +2,7 @@
 from django.db.models.signals import post_save
 
 from onadata.apps.logger.models.xform import XForm
+from onadata.libs.utils.model_tools import set_uuid
 
 
 class MergedXForm(XForm):
@@ -15,6 +16,10 @@ class MergedXForm(XForm):
     class Meta:
         app_label = 'logger'
 
+    def save(self, *args, **kwargs):
+        set_uuid(self)
+        return super(MergedXForm, self).save(*args, **kwargs)
+
 
 def set_object_permissions(sender, instance=None, created=False, **kwargs):
     if created:

onadata/libs/filters.py

@@ -1,3 +1,5 @@
+from uuid import UUID
+
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ObjectDoesNotExist
@@ -29,18 +31,28 @@ def filter_queryset(self, request, queryset, view):
         if request.user.is_anonymous:
             return queryset
 
-        if form_id and lookup_field == 'pk':
-            int_or_parse_error(form_id, u'Invalid form ID: %s')
         if form_id:
-            xform_kwargs = {lookup_field: form_id}
-            # check if form is public and return it
+            if lookup_field == 'pk':
+                int_or_parse_error(form_id, u'Invalid form ID: %s')
+
             try:
-                form = queryset.get(**xform_kwargs)
+                if lookup_field == 'uuid':
+                    form_id = UUID(form_id)
+                    form = queryset.get(
+                        Q(uuid=form_id.hex) | Q(uuid=str(form_id)))
+                else:
+                    xform_kwargs = {lookup_field: form_id}
+                    form = queryset.get(**xform_kwargs)
             except ObjectDoesNotExist:
                 raise Http404
 
+            # Check if form is public and return it
             if form.shared:
-                return queryset.filter(Q(**xform_kwargs))
+                if lookup_field == 'uuid':
+                    return queryset.filter(
+                        Q(uuid=form_id.hex) | Q(uuid=str(form_id)))
+                else:
+                    return queryset.filter(Q(**xform_kwargs))
 
         return super(AnonDjangoObjectPermissionFilter, self)\
             .filter_queryset(request, queryset, view)

onadata/libs/serializers/floip_serializer.py

@@ -5,13 +5,15 @@
 """
 import json
 import os
+from uuid import UUID
 from copy import deepcopy
 from io import BytesIO
 
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.exceptions import ValidationError
 from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.db.models import Q
 from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext_lazy as _
 
@@ -78,14 +80,22 @@ def parse_responses(responses, session_id_index=SESSION_ID_INDEX,
     yield submission
 
 
+class ReadOnlyUUIDField(serializers.ReadOnlyField):
+    """
+    Custom ReadOnlyField for UUID
+    """
+    def to_representation(self, obj):  # pylint: disable=no-self-use
+        return str(UUID(obj))
+
+
 # pylint: disable=too-many-ancestors
 class FloipListSerializer(serializers.HyperlinkedModelSerializer):
     """
     FloipListSerializer class.
     """
     url = serializers.HyperlinkedIdentityField(
         view_name='flow-results-detail', lookup_field='uuid')
-    id = serializers.ReadOnlyField(source='uuid')  # pylint: disable=C0103
+    id = ReadOnlyUUIDField(source='uuid')  # pylint: disable=C0103
     name = serializers.ReadOnlyField(source='id_string')
     created = serializers.ReadOnlyField(source='date_created')
     modified = serializers.ReadOnlyField(source='date_modified')
@@ -186,10 +196,11 @@ def update(self, instance, validated_data):
 
     def to_representation(self, instance):
         request = self.context['request']
+        data_id = str(UUID(instance.uuid))
         data_url = request.build_absolute_uri(
-            reverse('flow-results-responses', kwargs={'uuid': instance.uuid}))
+            reverse('flow-results-responses', kwargs={'uuid': data_id}))
         package = survey_to_floip_package(
-            json.loads(instance.json), instance.uuid, instance.date_created,
+            json.loads(instance.json), data_id, instance.date_created,
             instance.date_modified, data_url)
 
         data = package.descriptor
@@ -232,8 +243,11 @@ def create(self, validated_data):
         duplicates = 0
         request = self.context['request']
         responses = validated_data['responses']
-        xform = get_object_or_404(XForm, uuid=validated_data['id'],
-                                  deleted_at__isnull=True)
+        uuid = UUID(validated_data['id'])
+        xform = get_object_or_404(
+            XForm,
+            Q(uuid=str(uuid)) | Q(uuid=uuid.hex),
+            deleted_at__isnull=True)
         for submission in parse_responses(responses):
             xml_file = BytesIO(dict2xform(
                 submission, xform.id_string, 'data').encode('utf-8'))

onadata/libs/serializers/merged_xform_serializer.py

@@ -150,7 +150,8 @@ class MergedXFormSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = MergedXForm
         fields = ('url', 'id', 'xforms', 'name', 'project', 'title',
-                  'num_of_submissions', 'last_submission_time')
+                  'num_of_submissions', 'last_submission_time', 'uuid')
+        write_only_fields = ('uuid', )
 
     # pylint: disable=no-self-use
     def get_num_of_submissions(self, obj):

requirements/base.in

@@ -5,6 +5,6 @@
 -e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest
 -e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest
 -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router
--e git+https://github.com/onaio/floip-py.git@3bbf5c76b34ec49c438a3099ab848870514d1e50#egg=pyfloip
+-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip
 -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient
 -e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client

requirements/base.pip

@@ -7,7 +7,7 @@
 -e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest  # via -r requirements/base.in
 -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router  # via -r requirements/base.in
 -e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client  # via -r requirements/base.in
--e git+https://github.com/onaio/floip-py.git@3bbf5c76b34ec49c438a3099ab848870514d1e50#egg=pyfloip  # via -r requirements/base.in
+-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip  # via -r requirements/base.in
 -e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest  # via -r requirements/base.in
 -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient  # via -r requirements/base.in
 alabaster==0.7.12         # via sphinx

requirements/dev.pip

@@ -7,7 +7,7 @@
 -e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest  # via -r requirements/base.in
 -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router  # via -r requirements/base.in
 -e git+https://github.com/onaio/oauth2client.git@75dfdee77fb640ae30469145c66440571dfeae5c#egg=oauth2client  # via -r requirements/base.in
--e git+https://github.com/onaio/floip-py.git@3bbf5c76b34ec49c438a3099ab848870514d1e50#egg=pyfloip  # via -r requirements/base.in
+-e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip  # via -r requirements/base.in
 -e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest  # via -r requirements/base.in
 -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient  # via -r requirements/base.in
 alabaster==0.7.12         # via sphinx