Use chunked queryset when iterating queryset (#2701)

* use chunked queryset when iterating queryset

* use chunked queryset when iterating queryset

onadata/apps/api/tools.py

@@ -217,7 +217,9 @@ def remove_user_from_organization(organization, user):
         role_cls.remove_obj_permissions(user, organization.userprofile_ptr)
 
     # Remove user from all org projects
-    for project in organization.user.project_org.all():
+    project_qs = organization.user.project_org.all()
+
+    for project in queryset_iterator(project_qs):
         ShareProject(project, user.username, role, remove=True).save()
 
 
@@ -690,7 +692,9 @@ def get_xform_users(xform):
     """
     data = {}
     org_members = []
-    for perm in xform.xformuserobjectpermission_set.all():
+    xform_user_obj_perm_qs = xform.xformuserobjectpermission_set.all()
+
+    for perm in queryset_iterator(xform_user_obj_perm_qs):
         if perm.user not in data:
             user = perm.user
 

onadata/libs/models/share_project.py

@@ -18,16 +18,19 @@
     PROJ_PERM_CACHE,
     safe_delete,
 )
+from onadata.libs.utils.model_tools import queryset_iterator
 from onadata.libs.utils.project_utils import propagate_project_permissions_async
 
+
 # pylint: disable=invalid-name
 User = get_user_model()
 
 
 def remove_xform_permissions(project, user, role):
     """Remove user permissions to all forms for the given ``project``."""
     # remove role from project forms as well
-    for xform in project.xform_set.all():
+    xform_qs = project.xform_set.all()
+    for xform in queryset_iterator(xform_qs):
         # pylint: disable=protected-access
         role._remove_obj_permissions(user, xform)
         # Removed MergedXForm permissions if XForm is also a MergedXForm
@@ -37,14 +40,16 @@ def remove_xform_permissions(project, user, role):
 
 def remove_dataview_permissions(project, user, role):
     """Remove user permissions to all dataviews for the given ``project``."""
-    for dataview in project.dataview_set.all():
+    dataview_qs = project.dataview_set.all()
+    for dataview in queryset_iterator(dataview_qs):
         # pylint: disable=protected-access
         role._remove_obj_permissions(user, dataview.xform)
 
 
 def remove_entity_list_permissions(project, user, role):
     """Remove user permissions for all entitylists for the given project"""
-    for entity_list in project.entity_lists.all():
+    entity_list_qs = project.entity_lists.all()
+    for entity_list in queryset_iterator(entity_list_qs):
         # pylint: disable=protected-access
         role._remove_obj_permissions(user, entity_list)
 
@@ -77,7 +82,8 @@ def save(self, **kwargs):
                 role.add(self.user, self.project)
 
                 # apply same role to forms under the project
-                for xform in self.project.xform_set.all():
+                xform_qs = self.project.xform_set.all()
+                for xform in queryset_iterator(xform_qs):
                     # check if there is xform meta perms set
                     meta_perms = xform.metadata_set.filter(data_type="xform_meta_perms")
                     if meta_perms:
@@ -99,12 +105,14 @@ def save(self, **kwargs):
                     if hasattr(xform, "mergedxform"):
                         role.add(self.user, xform.mergedxform)
 
-                for dataview in self.project.dataview_set.all():
+                dataview_qs = self.project.dataview_set.all()
+                for dataview in queryset_iterator(dataview_qs):
                     if dataview.matches_parent:
                         role.add(self.user, dataview.xform)
 
                 # Apply same role to EntityLists under project
-                for entity_list in self.project.entity_lists.all():
+                entity_list_qs = self.project.entity_lists.all()
+                for entity_list in queryset_iterator(entity_list_qs):
                     role.add(self.user, entity_list)
 
         # clear cache