Add ability to permanently delete submissions

docs/data.rst

@@ -1099,6 +1099,55 @@ Response
     {"status_code": 200, "message": "3 records were deleted"}
 
 
+Permanent Deletion of Submissions
+------------------------------------
+
+**Permanently Delete a specific submission instance**
+
+`DELETE /api/v1/data/{pk}/{dataid}`
+
+A POST payload of parameter `permanent_delete` with the value 'True'. The value is 'False' by default.
+
+Note: This functionality is only enabled when the ``ENABLE_SUBMISSION_PERMANENT_DELETE`` setting is set to `True` within the application
+
+**Payload**
+::
+
+    permanent_delete = 'True'
+
+Example
+^^^^^^^^^
+::
+
+    `curl -X DELETE https://api.ona.io/api/v1/data/28058' -d 'permanent_delete=True'`
+
+Response
+^^^^^^^^^
+
+::
+    HTTP 204 No Content
+
+**Permanently Delete a subset of submissions**
+
+`DELETE /api/v1/data/{pk}`
+
+Example
+^^^^^^^^^
+::
+
+    `curl -X DELETE https://api.ona.io/api/v1/data/28058' -d 'permanent_delete=True' -d 'instance_ids=101425,108428,1974624'`
+
+Response
+^^^^^^^^^
+
+::
+
+    {
+        "status_code": "200",
+        "message": "3 records were deleted"
+    }
+
+
 GEOJSON
 -------
 

onadata/apps/api/tests/viewsets/test_data_viewset.py

@@ -1729,6 +1729,174 @@ def test_deletion_of_bulk_submissions(self, send_message_mock):
         self.assertEqual(current_count, 2)
         self.assertEqual(self.xform.num_of_submissions, 2)
 
+    @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=True)
+    @patch("onadata.apps.api.viewsets.data_viewset.send_message")
+    def test_submissions_permanent_deletion(self, send_message_mock):
+        """
+        Test that permanent submission deletions work
+        """
+        self._make_submissions()
+        self.xform.refresh_from_db()
+        formid = self.xform.pk
+        dataid = self.xform.instances.all().order_by("id")[0].pk
+        view = DataViewSet.as_view({"delete": "destroy", "get": "list"})
+
+        # initial count = 4 submissions
+        request = self.factory.get("/", **self.extra)
+        response = view(request, pk=formid)
+        self.assertEqual(len(response.data), 4)
+
+        request = self.factory.delete(
+            "/", **self.extra, data={"permanent_delete": True}
+        )
+        response = view(request, pk=formid, dataid=dataid)
+        self.assertEqual(response.status_code, 204)
+
+        # test that xform submission count is updated
+        self.xform.refresh_from_db()
+        self.assertEqual(self.xform.num_of_submissions, 3)
+        self.assertEqual(self.xform.instances.count(), 3)
+
+        # Test project details updated successfully
+        self.assertEqual(
+            self.xform.project.date_modified.strftime("%Y-%m-%d %H:%M:%S"),
+            timezone.now().strftime("%Y-%m-%d %H:%M:%S"),
+        )
+
+        # message sent upon delete
+        self.assertTrue(send_message_mock.called)
+        send_message_mock.assert_called_with(
+            instance_id=dataid,
+            target_id=formid,
+            target_type=XFORM,
+            user=request.user,
+            message_verb=SUBMISSION_DELETED,
+        )
+
+        # second delete of same submission should return 404
+        request = self.factory.delete(
+            "/", **self.extra, data={"permanent_delete": True}
+        )
+        response = view(request, pk=formid, dataid=dataid)
+        self.assertEqual(response.status_code, 404)
+
+        # remaining 3 submissions
+        request = self.factory.get("/", **self.extra)
+        response = view(request, pk=formid)
+        self.assertEqual(len(response.data), 3)
+
+        # check number of instances and num_of_submissions field
+        self.assertEqual(self.xform.instances.count(), 3)
+        self.assertEqual(self.xform.num_of_submissions, 3)
+
+    @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=True)
+    @patch("onadata.apps.api.viewsets.data_viewset.send_message")
+    def test_permanent_deletions_bulk_submissions(self, send_message_mock):
+        """
+        Test that permanent bulk submission deletions work
+        """
+        self._make_submissions()
+        self.xform.refresh_from_db()
+
+        formid = self.xform.pk
+        initial_count = self.xform.num_of_submissions
+        view = DataViewSet.as_view({"delete": "destroy"})
+
+        # test with valid instance id's
+        records_to_be_deleted = self.xform.instances.all()[:2]
+        instance_ids = ",".join([str(i.pk) for i in records_to_be_deleted])
+        data = {"instance_ids": instance_ids, "permanent_delete": True}
+
+        request = self.factory.delete("/", data=data, **self.extra)
+        response = view(request, pk=formid)
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(
+            response.data.get("message"),
+            "%d records were deleted" % len(records_to_be_deleted),
+        )
+        self.assertTrue(send_message_mock.called)
+        send_message_mock.called_with(
+            [str(i.pk) for i in records_to_be_deleted],
+            formid,
+            XFORM,
+            request.user,
+            SUBMISSION_DELETED,
+        )
+        self.xform.refresh_from_db()
+        current_count = self.xform.num_of_submissions
+        self.assertNotEqual(current_count, initial_count)
+        self.assertEqual(current_count, 2)
+        self.assertEqual(self.xform.num_of_submissions, 2)
+
+        # check number of xform instances
+        self.assertEqual(self.xform.instances.count(), 2)
+
+    @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=True)
+    @patch("onadata.apps.api.viewsets.data_viewset.send_message")
+    def test_permanent_instance_delete_inactive_form(self, send_message_mock):
+        """
+        Test that permanent submission deletions works on inactive forms
+        """
+        self._make_submissions()
+        formid = self.xform.pk
+        dataid = self.xform.instances.all().order_by("id")[0].pk
+        view = DataViewSet.as_view(
+            {
+                "delete": "destroy",
+            }
+        )
+
+        request = self.factory.delete(
+            "/", **self.extra, data={"permanent_delete": True}
+        )
+        response = view(request, pk=formid, dataid=dataid)
+
+        self.assertEqual(response.status_code, 204)
+
+        # test that xform submission count is updated
+        self.xform.refresh_from_db()
+        self.assertEqual(self.xform.num_of_submissions, 3)
+        self.assertEqual(self.xform.instances.count(), 3)
+
+        # make form inactive
+        self.xform.downloadable = False
+        self.xform.save()
+
+        dataid = self.xform.instances.filter(deleted_at=None).order_by("id")[0].pk
+
+        request = self.factory.delete("/", **self.extra, data={"permanent_delete": True})
+        response = view(request, pk=formid, dataid=dataid)
+
+        self.assertEqual(response.status_code, 204)
+
+        # test that xform submission count is updated
+        self.xform.refresh_from_db()
+        self.assertEqual(self.xform.num_of_submissions, 2)
+        self.assertTrue(send_message_mock.called)
+
+        # check number of instances and num_of_submissions field
+        self.assertEqual(self.xform.instances.count(), 2)
+
+    @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=False)
+    def test_failed_permanent_deletion(self):
+        """
+        Test that permanent submission deletion throws bad request when
+        functionality is disabled
+        """
+        self._make_submissions()
+        formid = self.xform.pk
+        dataid = self.xform.instances.all().order_by("id")[0].pk
+        view = DataViewSet.as_view({"delete": "destroy"})
+
+        request = self.factory.delete(
+            "/", **self.extra, data={"permanent_delete": True}
+        )
+        response = view(request, pk=formid, dataid=dataid)
+        self.assertEqual(response.status_code, 400)
+        error_msg = "Permanent submission deletion is not enabled for this server."
+        self.assertEqual(response.data, {"error": error_msg})
+
     @patch("onadata.apps.api.viewsets.data_viewset.send_message")
     def test_delete_submission_inactive_form(self, send_message_mock):
         self._make_submissions()

onadata/apps/api/viewsets/data_viewset.py

@@ -66,7 +66,7 @@
     OSMSerializer,
 )
 from onadata.libs.utils.api_export_tools import custom_response_handler
-from onadata.libs.utils.common_tools import json_stream
+from onadata.libs.utils.common_tools import json_stream, str_to_bool
 from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url
 
 SAFE_METHODS = ["GET", "HEAD", "OPTIONS"]
@@ -337,10 +337,19 @@ def enketo(self, request, *args, **kwargs):
 
         return Response(data=data)
 
+    # pylint: disable=too-many-branches,too-many-locals
     def destroy(self, request, *args, **kwargs):
-        """Soft deletes submissions data."""
+        """Deletes submissions data."""
         instance_ids = request.data.get("instance_ids")
         delete_all_submissions = strtobool(request.data.get("delete_all", "False"))
+        # get param to trigger permanent submission deletion
+        permanent_delete = str_to_bool(request.data.get("permanent_delete"))
+        enable_submission_permanent_delete = getattr(
+            settings, "ENABLE_SUBMISSION_PERMANENT_DELETE", False
+        )
+        permanent_delete_disabled_msg = _(
+            "Permanent submission deletion is not enabled for this server."
+        )
         # pylint: disable=attribute-defined-outside-init
         self.object = self.get_object()
 
@@ -364,8 +373,21 @@ def destroy(self, request, *args, **kwargs):
                     deleted_at__isnull=True,
                 )
 
+            error_msg = None
             for instance in queryset.iterator():
-                delete_instance(instance, request.user)
+                if permanent_delete:
+                    if enable_submission_permanent_delete:
+                        instance.delete()
+                    else:
+                        error_msg = {"error": permanent_delete_disabled_msg}
+                        break
+                else:
+                    # enable soft deletion
+                    delete_instance(instance, request.user)
+
+            if error_msg:
+                # return error msg if permanent deletion not enabled
+                return Response(error_msg, status=status.HTTP_400_BAD_REQUEST)
 
             # updates the num_of_submissions for the form.
             after_count = self.object.submission_count(force_update=True)
@@ -393,7 +415,22 @@ def destroy(self, request, *args, **kwargs):
 
             if request.user.has_perm(CAN_DELETE_SUBMISSION, self.object.xform):
                 instance_id = self.object.pk
-                delete_instance(self.object, request.user)
+                if permanent_delete:
+                    if enable_submission_permanent_delete:
+                        self.object.delete()
+                    else:
+                        error_msg = {"error": permanent_delete_disabled_msg}
+                        return Response(error_msg, status=status.HTTP_400_BAD_REQUEST)
+                else:
+                    # enable soft deletion
+                    delete_instance(self.object, request.user)
+
+                # updates the num_of_submissions for the form.
+                self.object.xform.submission_count(force_update=True)
+
+                # update the date modified field of the project
+                self.object.xform.project.date_modified = timezone.now()
+                self.object.xform.project.save(update_fields=["date_modified"])
 
                 # send message
                 send_message(