From 3d703342e53dc7949185c968b6046d2acb48dd33 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 20 Apr 2020 12:20:07 +0300 Subject: [PATCH 1/4] Update replaced attachment test Update test_replaced_attachments_not_tracked test making sure that file uploads are properly handled and the soft_deletion feature works as expected --- onadata/libs/tests/utils/test_logger_tools.py | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/onadata/libs/tests/utils/test_logger_tools.py b/onadata/libs/tests/utils/test_logger_tools.py index ad0f8be87b..e8f227467d 100644 --- a/onadata/libs/tests/utils/test_logger_tools.py +++ b/onadata/libs/tests/utils/test_logger_tools.py @@ -247,7 +247,8 @@ def test_replaced_attachments_not_tracked(self): md = """ | survey | | | | | | type | name | label | - | | image | image1 | Photo | + | | file | file | File | + | | image | image | Image | """ self._create_user_and_login() self.xform = self._publish_markdown(md, self.user) @@ -257,21 +258,28 @@ def test_replaced_attachments_not_tracked(self): uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7 - 1300221157303.jpg + Health_2011_03_13.xml_2011-03-15_20-30-28.xml + 1300221157303.jpg """.format(self.xform.id_string) - file_path = "{}/apps/logger/tests/Health_2011_03_13."\ - "xml_2011-03-15_20-30-28/1300221157303"\ - ".jpg".format(settings.PROJECT_ROOT) - media_file = django_file( - path=file_path, field_name="image1", content_type="image/jpeg") + media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' + '_2011_03_13.xml_2011-03-15_20-30-28/') + image_media = django_file( + path=f'{media_root}1300221157303.jpg', field_name='image', + content_type='image/jpeg') + file_media = django_file( + path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', + field_name='file', content_type='text/xml') instance = create_instance( self.user.username, BytesIO(xml_string.strip().encode('utf-8')), - media_files=[media_file]) + media_files=[file_media, image_media]) self.assertTrue(instance.json[MEDIA_ALL_RECEIVED]) - self.assertEquals(instance.json[TOTAL_MEDIA], 1) - self.assertEquals(instance.json[MEDIA_COUNT], 1) + self.assertEqual( + instance.attachments.filter(deleted_at__isnull=True).count(), + 2) + self.assertEquals(instance.json[TOTAL_MEDIA], 2) + self.assertEquals(instance.json[MEDIA_COUNT], 2) self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], @@ -304,6 +312,7 @@ def test_replaced_attachments_not_tracked(self): instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) # Test that only one attachment is recognised for this submission + # Since the file is no longer present in the submission self.assertEquals(instance2.json[TOTAL_MEDIA], 1) self.assertEquals(instance2.json[MEDIA_COUNT], 1) self.assertEquals( From ca8b59cd7308d45358a1b154aad78bb9a586f9fa Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 20 Apr 2020 12:33:57 +0300 Subject: [PATCH 2/4] Add file to the KNOWN_MEDIA_TYPES list --- onadata/libs/utils/common_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index de7bbfa669..3147a2c7df 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -90,7 +90,7 @@ OWNER_TEAM_NAME = "Owners" API_TOKEN = 'api-token' -KNOWN_MEDIA_TYPES = ['photo', 'image', 'audio', 'video'] +KNOWN_MEDIA_TYPES = ['photo', 'image', 'audio', 'video', 'file'] MEDIA_FILE_TYPES = { "image": ["image/png", "image/jpeg", "image/jpg"], "audio": ["audio/mp3", "audio/mp4"], From b5c56efc3c2ad63475a2c9622aaa397d5cede4f5 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 20 Apr 2020 16:59:48 +0300 Subject: [PATCH 3/4] Add management command to recover deleted attachments --- .../commands/recover_deleted_attachments.py | 55 +++++++++++++++ .../apps/logger/tests/management/__init__.py | 0 .../tests/management/commands/__init__.py | 0 .../test_recover_deleted_attachments.py | 68 +++++++++++++++++++ 4 files changed, 123 insertions(+) create mode 100644 onadata/apps/logger/management/commands/recover_deleted_attachments.py create mode 100644 onadata/apps/logger/tests/management/__init__.py create mode 100644 onadata/apps/logger/tests/management/commands/__init__.py create mode 100644 onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py diff --git a/onadata/apps/logger/management/commands/recover_deleted_attachments.py b/onadata/apps/logger/management/commands/recover_deleted_attachments.py new file mode 100644 index 0000000000..fa1bc6629f --- /dev/null +++ b/onadata/apps/logger/management/commands/recover_deleted_attachments.py @@ -0,0 +1,55 @@ +""" +Module containing the recover_deleted_attachments management command. + +Used to recover attachments that were accidentally deleted within the system +but are still required/present within the submission XML + +Sample usage: python manage.py recover_deleted_attachments --form 1 +""" +from django.core.management.base import BaseCommand + +from onadata.apps.logger.models import Instance + + +def recover_deleted_attachments(form_id: str, stdout=None): + """ + Recovers attachments that were accidentally soft-deleted + + :param: (str) form_id: Unique identifier for an XForm object + :param: (sys.stdout) stdout: Python standard output. Default: None + """ + instances = Instance.objects.filter( + xform__id=form_id, deleted_at__isnull=True) + for instance in instances: + expected_attachments = instance.get_expected_media() + if not instance.attachments.filter( + deleted_at__isnull=True).count() == len(expected_attachments): + attachments_to_recover = instance.attachments.filter( + deleted_at__isnull=False, + name__in=expected_attachments) + for attachment in attachments_to_recover: + attachment.deleted_at = None + attachment.deleted_by = None + attachment.save() + + if stdout: + stdout.write( + f'Recovered {attachment.name} ID: {attachment.id}') + # Regenerate instance JSON + instance.json = instance.get_full_dict(load_existing=False) + instance.save() + + +class Command(BaseCommand): + """ + Management command used to recover wrongfully deleted + attachments. + """ + help = 'Restore wrongly deleted attachments' + + def add_arguments(self, parser): + parser.add_argument('-f', '--form', dest='form_id', type=int) + + def handle(self, *args, **options): + form_id = options.get('form_id') + recover_deleted_attachments(form_id, self.stdout) diff --git a/onadata/apps/logger/tests/management/__init__.py b/onadata/apps/logger/tests/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/onadata/apps/logger/tests/management/commands/__init__.py b/onadata/apps/logger/tests/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py b/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py new file mode 100644 index 0000000000..2d07ab317c --- /dev/null +++ b/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py @@ -0,0 +1,68 @@ +from io import BytesIO +from datetime import datetime + +from django.conf import settings + +from onadata.apps.main.tests.test_base import TestBase +from onadata.apps.logger.import_tools import django_file +from onadata.apps.logger.management.commands.recover_deleted_attachments \ + import recover_deleted_attachments +from onadata.libs.utils.logger_tools import create_instance + + +class TestRecoverDeletedAttachments(TestBase): + def test_recovers_wrongly_deleted_attachments(self): + """ + Test that the command recovers the correct + attachment + """ + md = """ + | survey | | | | + | | type | name | label | + | | file | file | File | + | | image | image | Image | + """ + self._create_user_and_login() + self.xform = self._publish_markdown(md, self.user) + + xml_string = f""" + + + uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7 + + Health_2011_03_13.xml_2011-03-15_20-30-28.xml + 1300221157303.jpg + + """ + media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' + '_2011_03_13.xml_2011-03-15_20-30-28/') + image_media = django_file( + path=f'{media_root}1300221157303.jpg', field_name='image', + content_type='image/jpeg') + file_media = django_file( + path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', + field_name='file', content_type='text/xml') + instance = create_instance( + self.user.username, + BytesIO(xml_string.strip().encode('utf-8')), + media_files=[file_media, image_media]) + self.assertEqual( + instance.attachments.filter(deleted_at__isnull=True).count(), 2) + attachment = instance.attachments.first() + + # Soft delete attachment + attachment.deleted_at = datetime.now() + attachment.deleted_by = self.user + attachment.save() + + self.assertEqual( + instance.attachments.filter(deleted_at__isnull=True).count(), 1) + + # Attempt recovery of attachment + recover_deleted_attachments(form_id=instance.xform.id) + + self.assertEqual( + instance.attachments.filter(deleted_at__isnull=True).count(), 2) + attachment.refresh_from_db() + self.assertIsNone(attachment.deleted_at) + self.assertIsNone(attachment.deleted_by) From 74d003b2e1cae3e65d9177fe94ea636e3d35db07 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Mon, 4 May 2020 11:28:35 +0300 Subject: [PATCH 4/4] Fix pylint errors --- .../commands/test_recover_deleted_attachments.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py b/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py index 2d07ab317c..03ba6bb992 100644 --- a/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py +++ b/onadata/apps/logger/tests/management/commands/test_recover_deleted_attachments.py @@ -1,3 +1,7 @@ +""" +Module containing the tests for the recover_deleted_attachments +management command +""" from io import BytesIO from datetime import datetime @@ -11,6 +15,8 @@ class TestRecoverDeletedAttachments(TestBase): + """TestRecoverDeletedAttachments Class""" + # pylint: disable=invalid-name def test_recovers_wrongly_deleted_attachments(self): """ Test that the command recovers the correct @@ -21,12 +27,12 @@ def test_recovers_wrongly_deleted_attachments(self): | | type | name | label | | | file | file | File | | | image | image | Image | - """ + """ # pylint: disable=invalid-name self._create_user_and_login() - self.xform = self._publish_markdown(md, self.user) + xform = self._publish_markdown(md, self.user) xml_string = f""" - + uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7