1188 remove x total

onadata/apps/api/tests/viewsets/test_data_viewset.py

@@ -441,31 +441,23 @@ def test_data_pagination(self):
         request = self.factory.get('/', **self.extra)
         response = view(request, pk=formid)
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.has_header('X-total'))
-        self.assertEqual(int(response.get('X-total')), 4)
         self.assertEqual(len(response.data), 4)
 
         request = self.factory.get('/', data={"page": "1", "page_size": 2},
                                    **self.extra)
         response = view(request, pk=formid)
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.has_header('X-total'))
-        self.assertEqual(int(response.get('X-total')), 4)
         self.assertEqual(len(response.data), 2)
 
         request = self.factory.get('/', data={"page_size": "3"}, **self.extra)
         response = view(request, pk=formid)
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.has_header('X-total'))
-        self.assertEqual(int(response.get('X-total')), 4)
         self.assertEqual(len(response.data), 3)
 
         request = self.factory.get(
             '/', data={"page": "1", "page_size": "2"}, **self.extra)
         response = view(request, pk=formid)
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.has_header('X-total'))
-        self.assertEqual(int(response.get('X-total')), 4)
         self.assertEqual(len(response.data), 2)
 
         # invalid page returns a 404
@@ -722,17 +714,13 @@ def test_data_public(self):
         request = self.factory.get('/', **self.extra)
         response = view(request, pk='public')
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.has_header('X-total'))
-        self.assertEqual(int(response.get('X-total')), 0)
         self.assertEqual(response.data, [])
         self.xform.shared_data = True
         self.xform.save()
         formid = self.xform.pk
         data = _data_list(formid)
         response = view(request, pk='public')
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.has_header('X-total'))
-        self.assertEqual(int(response.get('X-total')), 1)
         self.assertEqual(response.data, data)
 
     def test_data_public_anon_user(self):
@@ -870,8 +858,6 @@ def test_data_with_query_parameter(self):
         request = self.factory.get('/?query=%s' % query_str, **self.extra)
         response = view(request, pk=formid)
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.has_header('X-total'))
-        self.assertEqual(int(response.get('X-total')), 1)
         self.assertEqual(len(response.data), 1)
 
         submission_time = instance.date_created.strftime(MONGO_STRFTIME)

onadata/apps/api/viewsets/data_viewset.py

@@ -38,7 +38,6 @@
     AuthenticateHeaderMixin
 from onadata.libs.mixins.cache_control_mixin import CacheControlMixin
 from onadata.libs.mixins.etags_mixin import ETagsMixin
-from onadata.libs.mixins.total_header_mixin import TotalHeaderMixin
 from onadata.libs.pagination import StandardPageNumberPagination
 from onadata.libs.serializers.data_serializer import DataSerializer
 from onadata.libs.serializers.data_serializer import (
@@ -54,6 +53,7 @@
 from onadata.libs.utils.viewer_tools import get_enketo_edit_url
 from onadata.libs.utils.api_export_tools import custom_response_handler
 from onadata.libs.data import parse_int
+from onadata.libs.utils.common_tools import json_stream
 from onadata.apps.api.permissions import ConnectViewsetPermissions
 from onadata.apps.api.tools import get_baseviewset_class
 from onadata.apps.logger.models.instance import FormInactiveError
@@ -86,7 +86,6 @@ def delete_instance(instance):
 class DataViewSet(AnonymousUserPublicFormsMixin,
                   AuthenticateHeaderMixin,
                   ETagsMixin, CacheControlMixin,
-                  TotalHeaderMixin,
                   BaseViewset,
                   ModelViewSet):
     """
@@ -416,7 +415,7 @@ def list(self, request, *args, **kwargs):
 
         if export_type == Attachment.OSM:
             if request.GET:
-                self.set_object_list_and_total_count(
+                self.set_object_list(
                     query, fields, sort, start, limit, is_public_request)
                 kwargs = {'instance__in': self.object_list}
             osm_list = OsmData.objects.filter(**kwargs)
@@ -436,7 +435,7 @@ def list(self, request, *args, **kwargs):
 
         return custom_response_handler(request, xform, query, export_type)
 
-    def set_object_list_and_total_count(
+    def set_object_list(
             self, query, fields, sort, start, limit, is_public_request):
         try:
             if not is_public_request:
@@ -452,9 +451,7 @@ def set_object_list_and_total_count(
                 limit = limit if start is None or start == 0 else start + limit
                 self.object_list = filter_queryset_xform_meta_perms(
                     self.get_object(), self.request.user, self.object_list)
-                self.object_list = \
-                    self.object_list.order_by('pk')[start: limit]
-                self.total_count = self.object_list.count()
+                self.object_list = self.object_list[start:limit]
             elif (sort or limit or start or fields) and not is_public_request:
                 try:
                     query = \
@@ -464,21 +461,12 @@ def set_object_list_and_total_count(
                     self.object_list = query_data(xform, query=query,
                                                   sort=sort, start_index=start,
                                                   limit=limit, fields=fields)
-                    self.total_count = query_data(
-                        xform, query=query, sort=sort, start_index=start,
-                        limit=limit, fields=fields, count=True
-                    )[0].get('count')
-
                 except NoRecordsPermission:
                     self.object_list = []
-                    self.total_count = 0
 
-            else:
-                self.total_count = self.object_list.count()
-
-            if self.total_count and isinstance(self.object_list, QuerySet):
+            if isinstance(self.object_list, QuerySet):
                 self.etag_hash = get_etag_hash_from_query(self.object_list)
-            elif self.total_count:
+            else:
                 sql, params, records = get_sql_with_params(
                     xform, query=query, sort=sort, start_index=start,
                     limit=limit, fields=fields
@@ -490,7 +478,7 @@ def set_object_list_and_total_count(
             raise ParseError(unicode(e))
 
     def _get_data(self, query, fields, sort, start, limit, is_public_request):
-        self.set_object_list_and_total_count(
+        self.set_object_list(
             query, fields, sort, start, limit, is_public_request)
 
         pagination_keys = [self.paginator.page_query_param,
@@ -503,34 +491,23 @@ def _get_data(self, query, fields, sort, start, limit, is_public_request):
 
         STREAM_DATA = getattr(settings, 'STREAM_DATA', False)
         if STREAM_DATA:
-            length = self.total_count
-            if should_paginate and \
-                    not isinstance(self.object_list, types.GeneratorType):
-                length = len(self.object_list)
-            response = self._get_streaming_response(length)
+            response = self._get_streaming_response()
         else:
             serializer = self.get_serializer(self.object_list, many=True)
             response = Response(serializer.data)
 
         return response
 
-    def _get_streaming_response(self, length):
-        """Get a StreamingHttpResponse response object
-
-        @param length ensures a valid JSON is generated, avoid a trailing comma
+    def _get_streaming_response(self):
         """
-        def stream_json(data, length):
-            """Generator function to stream JSON data"""
-            yield u"["
-
-            for i, d in enumerate(data, start=1):
-                yield json.dumps(d.json if isinstance(d, Instance) else d)
-                yield "" if i == length else ","
-
-            yield u"]"
+        Get a StreamingHttpResponse response object
+        """
+        def get_json_string(item):
+            return json.dumps(
+                item.json if isinstance(item, Instance) else item)
 
         response = StreamingHttpResponse(
-            stream_json(self.object_list, length),
+            json_stream(self.object_list, get_json_string),
             content_type="application/json"
         )
 

onadata/apps/api/viewsets/open_data_viewset.py

@@ -18,10 +18,10 @@
 from onadata.libs.data import parse_int
 from onadata.libs.mixins.cache_control_mixin import CacheControlMixin
 from onadata.libs.mixins.etags_mixin import ETagsMixin
-from onadata.libs.mixins.total_header_mixin import TotalHeaderMixin
 from onadata.libs.pagination import StandardPageNumberPagination
 from onadata.libs.serializers.data_serializer import DataInstanceSerializer
 from onadata.libs.serializers.open_data_serializer import OpenDataSerializer
+from onadata.libs.utils.common_tools import json_stream
 from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
 
 BaseViewset = get_baseviewset_class()
@@ -32,7 +32,7 @@ def replace_special_characters_with_underscores(data):
     return [re.sub(r"\W", r"_", a) for a in data]
 
 
-class OpenDataViewSet(ETagsMixin, CacheControlMixin, TotalHeaderMixin,
+class OpenDataViewSet(ETagsMixin, CacheControlMixin,
                       BaseViewset, ModelViewSet):
     permission_classes = (OpenDataViewSetPermissions, )
     queryset = OpenData.objects.filter()
@@ -132,42 +132,33 @@ def data(self, request, **kwargs):
                 qs_kwargs.update({'id__gt': gt_id})
 
             instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
-            length = self.total_count = instances.count()
 
             if count:
-                return Response({'count': self.total_count})
+                return Response({'count': instances.count()})
 
             if should_paginate:
                 instances = self.paginate_queryset(instances)
-                length = 1 + self.paginator.page.end_index(
-                ) - self.paginator.page.start_index()
 
             csv_df_builder = CSVDataFrameBuilder(
                 xform.user.username, xform.id_string, include_images=False)
             data = csv_df_builder._format_for_dataframe(
                 DataInstanceSerializer(instances, many=True).data)
 
-            return self._get_streaming_response(data, length)
+            return self._get_streaming_response(data)
 
         return Response(data)
 
-    def _get_streaming_response(self, data, length):
+    def _get_streaming_response(self, data):
         """Get a StreamingHttpResponse response object"""
 
-        def stream_json(streaming_data, length):
-            """Generator function to stream JSON data"""
-            yield u"["
-
-            for i, d in enumerate(streaming_data, start=1):
-                yield json.dumps({
-                    re.sub(r"\W", r"_", a): b for a, b in d.items()
-                })
-                yield "" if i == length else ","
-
-            yield u"]"
+        def get_json_string(item):
+            return json.dumps({
+                re.sub(r"\W", r"_", a): b for a, b in item.items()})
 
         response = StreamingHttpResponse(
-            stream_json(data, length), content_type="application/json")
+            json_stream(data, get_json_string),
+            content_type="application/json"
+        )
 
         # set headers on streaming response
         for k, v in self.headers.items():

onadata/apps/api/viewsets/xform_viewset.py

@@ -62,6 +62,7 @@
                                                  get_async_response,
                                                  process_async_export,
                                                  response_for_format)
+from onadata.libs.utils.common_tools import json_stream
 from onadata.libs.utils.csv_import import (get_async_csv_submission_status,
                                            submit_csv, submit_csv_async)
 from onadata.libs.utils.export_tools import parse_request_export_options
@@ -681,31 +682,23 @@ def export_async(self, request, *args, **kwargs):
                         status=status.HTTP_202_ACCEPTED,
                         content_type="application/json")
 
-    def _get_streaming_response(self, length):
-        """Get a StreamingHttpResponse response object
-
-        @param length ensures a valid JSON is generated, avoid a trailing comma
+    def _get_streaming_response(self):
+        """
+        Get a StreamingHttpResponse response object
         """
-        def stream_json(data, length):
-            """Generator function to stream JSON data"""
-            yield u"["
-            start = 1
-            # use queryset_iterator.  Will need to change this to the Django
-            # native .iterator() method when we upgrade to django version 2
-            # because in Django 2 .iterator() has support for chunk size
-            queryset = queryset_iterator(self.object_list, chunksize=2000)
-            for xform in queryset:
-                yield json.dumps(XFormBaseSerializer(
-                    instance=xform,
-                    context={'request': self.request}
+        # use queryset_iterator.  Will need to change this to the Django
+        # native .iterator() method when we upgrade to django version 2
+        # because in Django 2 .iterator() has support for chunk size
+        queryset = queryset_iterator(self.object_list, chunksize=2000)
+
+        def get_json_string(item):
+            return json.dumps(XFormBaseSerializer(
+                instance=item,
+                context={'request': self.request}
                 ).data)
-                yield "" if start == length else ","
-                start += 1
-
-            yield u"]"
 
         response = StreamingHttpResponse(
-            stream_json(self.object_list, length),
+            json_stream(queryset, get_json_string),
             content_type="application/json"
         )
 
@@ -731,7 +724,7 @@ def list(self, request, *args, **kwargs):
                 self.etag_data = last_modified[0].isoformat()
             if STREAM_DATA:
                 self.object_list = queryset
-                resp = self._get_streaming_response(length=queryset.count())
+                resp = self._get_streaming_response()
             else:
                 resp = super(XFormViewSet, self).list(request, *args, **kwargs)
         except XLSFormError, e:

onadata/apps/main/views.py

@@ -532,7 +532,6 @@ def api(request, username=None, id_string=None):
         response_text = ("%s(%s)" % (callback, response_text))
 
     response = HttpResponse(response_text, content_type='application/json')
-    response['X-total'] = total_records
     add_cors_headers(response)
 
     return response

onadata/apps/viewer/static/js/data_view.js

@@ -31,7 +31,9 @@
             pageSize: "limit"
         },
         parseState: function (resp, queryParams, state, options) {
-            return {totalRecords: parseInt(options.xhr.getResponseHeader("X-total"))};
+            // Removing X-total
+            // {totalRecords: parseInt(options.xhr.getResponseHeader("X-total"))};
+            return 0;
         },
         initialize: function (models, options) {
             // set the url

onadata/libs/utils/common_tools.py

@@ -104,3 +104,26 @@ def get_response_content(response):
         contents = response.content
 
     return contents
+
+
+def json_stream(data, json_string):
+    """
+    Generator function to stream JSON data
+    """
+    yield u"["
+    try:
+        data = data.__iter__()
+        item = data.next()
+        while item:
+            try:
+                next_item = data.next()
+                yield json_string(item)
+                yield ","
+                item = next_item
+            except StopIteration:
+                yield json_string(item)
+                break
+    except AttributeError:
+        pass
+    finally:
+        yield u"]"