Clean out Merged Datasets upon xform deletion. Clean out Form Media F…

…iles upon form deletion. Add documentation on clean up of filtered datasets

onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py

@@ -663,18 +663,6 @@ def test_md_data_viewset_deleted_form(self):
 
         # DataViewSet /data/[pk] endpoint, form_a deleted
         form_a.soft_delete()
-        response = data_view(request, pk=merged_dataset['id'])
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.data), 1)
-
-        fruit = [d['fruit'] for d in response.data]
-        expected_fruit = ['mango']
-        self.assertEqual(fruit, expected_fruit)
-
-        # DataViewSet /data/[pk]/[dataid] endpoint, form_a deleted
-        data_view = DataViewSet.as_view({
-            'get': 'retrieve',
-        })
         response = data_view(request, pk=merged_dataset['id'], dataid=dataid)
         self.assertEqual(response.status_code, 404)
 

onadata/apps/api/tests/viewsets/test_metadata_viewset.py

@@ -192,6 +192,35 @@ def test_delete_metadata(self):
             self.assertEqual(response.status_code, 204)
             self.assertEqual(count, MetaData.objects.count())
 
+    def test_delete_xform_deletes_media_metadata(self):
+        self._add_test_metadata()
+        self.view = MetaDataViewSet.as_view({'get': 'list'})
+        data = {'xform': self.xform.pk}
+        request = self.factory.get('/', data, **self.extra)
+        response = self.view(request)
+        meta_obj = self.xform.metadata_set.all()[0]
+        expected_metadata = {
+            'id': meta_obj.id,
+            'xform': self.xform.id,
+            'data_value': 'screenshot.png',
+            'data_type': meta_obj.data_type,
+            'data_file': f'http://localhost:8000/media/{meta_obj.data_file.name}',  # noqa
+            'data_file_type': 'image/png',
+            'media_url': None,
+            'file_hash': 'md5:09b9e5e3278ac2a43fce25681ffada85',
+            'url': f'http://testserver/api/v1/metadata/{meta_obj.id}',
+            'date_created': meta_obj.date_created
+            }
+        self.assertEqual(response.status_code, 200)
+        self.assertEquals(dict(response.data[0]), expected_metadata)
+
+        # Soft delete xform
+        self.xform.soft_delete()
+        # Confirm that all metadata was deleted
+        response2 = self.view(request)
+        self.assertEqual(response2.status_code, 200)
+        self.assertEquals(response2.data, [])
+
     def test_windows_csv_file_upload_to_metadata(self):
         data_value = 'transportation.csv'
         path = os.path.join(self.fixture_dir, data_value)

onadata/apps/api/viewsets/metadata_viewset.py

@@ -30,7 +30,8 @@ class MetaDataViewSet(AuthenticateHeaderMixin,
 
     content_negotiation_class = MediaFileContentNegotiation
     filter_backends = (filters.MetaDataFilter,)
-    queryset = MetaData.objects.select_related()
+    queryset = MetaData.objects.filter(
+                deleted_at__isnull=True).select_related()
     permission_classes = (MetaDataObjectPermissions,)
     renderer_classes = (
         renderers.JSONRenderer,

onadata/apps/logger/models/xform.py

@@ -953,8 +953,17 @@ def soft_delete(self, user=None):
             update_fields.append('deleted_by')
 
         self.save(update_fields=update_fields)
+        # Delete associated filtered datasets
         for dataview in self.dataview_set.all():
             dataview.soft_delete(user)
+        # Delete associated Merged-Datasets
+        for merged_dataset in self.mergedxform_ptr.filter(
+                deleted_at___isnull=True):
+            merged_dataset.soft_delete(user)
+        # Delete associated Form Media Files
+        for metadata in self.metadata_set.filter(
+                deleted_at___isnull=True):
+            metadata.soft_delete()
 
     def submission_count(self, force_update=False):
         if self.num_of_submissions == 0 or force_update:

onadata/apps/main/models/meta_data.py

@@ -8,6 +8,7 @@
 
 import requests
 from django.conf import settings
+from django.utils import timezone
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
@@ -209,6 +210,16 @@ def _set_hash(self):
 
         return ''
 
+    def soft_delete(self):
+        """
+        Return the soft deletion timestamp
+        Mark the MetaData as soft deleted,
+        by updating the deleted_at field.
+        """
+        soft_deletion_time = timezone.now()
+        self.deleted_at = soft_deletion_time
+        self.save()
+
     @staticmethod
     def public_link(content_object, data_value=None):
         data_type = 'public_link'