Add pagination to forms endpoint

docs/forms.rst

@@ -173,6 +173,23 @@ Request
 
     curl -X GET https://api.ona.io/api/v1/forms?owner=ona
 
+Get a paginated list of forms
+------------------------------
+Returns a list of JSON forms using page number and the number of items per page. Use the ``page`` parameter to specify page number and ``page_size`` parameter is used to set the custom page size.
+
+- ``page`` - Integer representing the page.
+- ``page_size`` - Integer representing the number of records that should be returned in a single page.
+
+.. raw:: html
+
+    <pre class="prettyprint">
+    <b>GET</b> /api/v1/forms.json?<code>page</code>=<code>1</code><code>page_size</code>=<code>10</code></pre>
+
+Request
+^^^^^^^
+::
+
+    curl -X GET https://api.ona.io/api/v1/forms.json?page=1&page_size=10
 
 Get Form Information
 ---------------------

onadata/apps/api/tests/viewsets/test_xform_viewset.py

@@ -328,6 +328,83 @@ def test_form_list_stream(self):
             self.assertNotEqual(response.get("Cache-Control"), None)
             self.assertEqual(response.status_code, 200)
 
+    def test_form_list_with_pagination(self):
+        view = XFormViewSet.as_view(
+            {
+                "get": "list",
+            }
+        )
+        with HTTMock(enketo_mock):
+            self._publish_xls_form_to_project()
+            form_path = os.path.join(
+                settings.PROJECT_ROOT,
+                "apps",
+                "main",
+                "tests",
+                "fixtures",
+                "transportation",
+                "transportation_different_id_string.xlsx",
+            )
+            self._publish_xls_form_to_project(xlsform_path=form_path)
+            # no page param no pagination
+            request = self.factory.get("/", **self.extra)
+            response = view(request)
+            self.assertNotEqual(response.get("Cache-Control"), None)
+            self.assertEqual(response.status_code, 200)
+            self.assertTrue(len(response.data), 2)
+
+            # test pagination
+            request = self.factory.get(
+                "/", data={"page": 1, "page_size": 1}, **self.extra
+            )
+            response = view(request)
+            self.assertEqual(response.status_code, 200)
+            # check that only one form is returned
+            self.assertEqual(len(response.data), 1)
+
+    @override_settings(STREAM_DATA=True)
+    def test_form_list_stream_with_pagination(self):
+        view = XFormViewSet.as_view(
+            {
+                "get": "list",
+            }
+        )
+        with HTTMock(enketo_mock):
+            self._publish_xls_form_to_project()
+            form_path = os.path.join(
+                settings.PROJECT_ROOT,
+                "apps",
+                "main",
+                "tests",
+                "fixtures",
+                "transportation",
+                "transportation_different_id_string.xlsx",
+            )
+            self._publish_xls_form_to_project(xlsform_path=form_path)
+            # no page param no pagination
+            request = self.factory.get("/", **self.extra)
+            response = view(request)
+            self.assertTrue(response.streaming)
+            streaming_data = json.loads(
+                "".join([i.decode("utf-8") for i in response.streaming_content])
+            )
+            self.assertNotEqual(response.get("Cache-Control"), None)
+            self.assertEqual(response.status_code, 200)
+            self.assertEqual(len(streaming_data), 2)
+
+            # test pagination
+            request = self.factory.get(
+                "/", data={"page": 1, "page_size": 1}, **self.extra
+            )
+            response = view(request)
+            self.assertTrue(response.streaming)
+            streaming_data = json.loads(
+                "".join([i.decode("utf-8") for i in response.streaming_content])
+            )
+            self.assertEqual(response.status_code, 200)
+            # check that only one form is returned
+            self.assertEqual(len(streaming_data), 1)
+
     def test_form_list_without_enketo_connection(self):
         self._publish_xls_form_to_project()
         request = self.factory.get("/", **self.extra)
@@ -1649,16 +1726,17 @@ def test_publish_invalid_xls_form(self):
             )
             self.assertEqual(response.data.get("text"), error_msg)
 
-    @patch.object(ModelViewSet, "list")
+    @patch("onadata.apps.api.viewsets.xform_viewset.XFormViewSet.list")
     def test_return_400_on_xlsform_error_on_list_action(self, mock_set_title):
         with HTTMock(enketo_mock):
-            error_msg = "Title shouldn't have an ampersand"
-            mock_set_title.side_effect = XLSFormError(error_msg)
-            request = self.factory.get("/", **self.extra)
-            response = self.view(request)
-            self.assertTrue(mock_set_title.called)
-            self.assertEqual(response.status_code, 400)
-            self.assertEqual(response.content.decode("utf-8"), error_msg)
+            with self.assertRaises(XLSFormError):
+                error_msg = "Title shouldn't have an ampersand"
+                mock_set_title.side_effect = XLSFormError(error_msg)
+                request = self.factory.get("/", **self.extra)
+                response = self.view(request)
+                self.assertTrue(mock_set_title.called)
+                self.assertEqual(response.status_code, 400)
+                self.assertEqual(response.content.decode("utf-8"), error_msg)
 
     def test_publish_invalid_xls_form_no_choices(self):
         view = XFormViewSet.as_view({"post": "create"})

onadata/apps/api/viewsets/xform_viewset.py

@@ -64,6 +64,7 @@
 from onadata.libs.mixins.cache_control_mixin import CacheControlMixin
 from onadata.libs.mixins.etags_mixin import ETagsMixin
 from onadata.libs.mixins.labels_mixin import LabelsMixin
+from onadata.libs.pagination import StandardPageNumberPagination
 from onadata.libs.renderers import renderers
 from onadata.libs.serializers.clone_xform_serializer import CloneXFormSerializer
 from onadata.libs.serializers.share_xform_serializer import ShareXFormSerializer
@@ -89,7 +90,6 @@
 )
 from onadata.libs.utils.export_tools import parse_request_export_options
 from onadata.libs.utils.logger_tools import publish_form
-from onadata.libs.utils.model_tools import queryset_iterator
 from onadata.libs.utils.string import str2bool
 from onadata.libs.utils.viewer_tools import (
     generate_enketo_form_defaults,
@@ -326,6 +326,7 @@ class XFormViewSet(
         )
     )
     serializer_class = XFormSerializer
+    pagination_class = StandardPageNumberPagination
     lookup_field = "pk"
     extra_lookup_fields = None
     permission_classes = [
@@ -573,7 +574,7 @@ def retrieve(self, request, *args, **kwargs):
 
             page = self.paginate_queryset(self.object_list)
             if page is not None:
-                serializer = self.get_pagination_serializer(page)
+                serializer = self.get_serializer(page, many=True)
             else:
                 serializer = self.get_serializer(self.object_list, many=True)
 
@@ -942,7 +943,7 @@ def _get_streaming_response(self):
         # use queryset_iterator.  Will need to change this to the Django
         # native .iterator() method when we upgrade to django version 2
         # because in Django 2 .iterator() has support for chunk size
-        queryset = queryset_iterator(self.object_list, chunksize=2000)
+        queryset = self.object_list.instance
 
         def get_json_string(item):
             return json.dumps(
@@ -971,19 +972,24 @@ def get_json_string(item):
     def list(self, request, *args, **kwargs):
         """List forms API endpoint `GET /api/v1/forms`."""
         stream_data = getattr(settings, "STREAM_DATA", False)
+        # pylint: disable=attribute-defined-outside-init
         try:
-            queryset = self.filter_queryset(self.get_queryset())
-            last_modified = queryset.values_list("date_modified", flat=True).order_by(
-                "-date_modified"
-            )
-            # pylint: disable=attribute-defined-outside-init
+            self.object_list = self.filter_queryset(self.get_queryset())
+            last_modified = self.object_list.values_list(
+                "date_modified", flat=True
+            ).order_by("-date_modified")
+            page = self.paginate_queryset(self.object_list)
             if last_modified:
                 self.etag_data = last_modified[0].isoformat()
+            if page:
+                self.object_list = self.get_serializer(page, many=True)
+            else:
+                self.object_list = self.get_serializer(self.object_list, many=True)
             if stream_data:
-                self.object_list = queryset
                 resp = self._get_streaming_response()
             else:
-                resp = super().list(request, *args, **kwargs)
+                serializer = self.object_list
+                resp = Response(serializer.data, status=status.HTTP_200_OK)
         except XLSFormError as e:
             resp = HttpResponseBadRequest(e)