From 19323ec59665c9cdb858e8d95be5eb01197cfd44 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Sat, 25 Jul 2020 15:25:41 +0300 Subject: [PATCH 1/3] Change root node for created submissions to data By default XLSForms set the default node to 'data' --- .../tests/viewsets/test_xform_submission_viewset.py | 10 ++++++++++ onadata/libs/serializers/data_serializer.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py index 994456e077..ee0d048a9c 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py @@ -556,6 +556,16 @@ def test_rapidpro_post_submission(self): self.assertContains(response, 'Successful submission', status_code=201) self.assertTrue(response.has_header('Date')) self.assertEqual(response['Location'], 'http://testserver/submission') + # InstanceID is returned as uuid: + # Retrieving the uuid without the prefix in order to retrieve + # the actual instance + uuid = response.data.get('instanceID').split(':')[1] + instance = Instance.objects.get(uuid=uuid) + expected_xml = ( + "orange" + "") + self.assertEqual(instance.xml, expected_xml) def test_legacy_rapidpro_post_submission(self): """ diff --git a/onadata/libs/serializers/data_serializer.py b/onadata/libs/serializers/data_serializer.py index 99c4e69808..59e247639c 100644 --- a/onadata/libs/serializers/data_serializer.py +++ b/onadata/libs/serializers/data_serializer.py @@ -46,7 +46,7 @@ def create_submission(request, username, data_dict, xform_id): """ Returns validated data object instances """ - xml_string = dict2xform(data_dict, xform_id) + xml_string = dict2xform(data_dict, xform_id, root='data') xml_file = BytesIO(xml_string.encode('utf-8')) error, instance = safe_create_instance(username, xml_file, [], None, From aff9675fe527568589bc7abb9082df76dba9fd71 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Tue, 28 Jul 2020 10:45:54 +0300 Subject: [PATCH 2/3] Retrieve root node name from XForm survey object --- .../viewsets/test_xform_submission_viewset.py | 9 ++--- onadata/libs/serializers/data_serializer.py | 3 +- onadata/libs/utils/logger_tools.py | 34 +++++++++++++++++-- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py index ee0d048a9c..d5ace9d464 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py @@ -556,16 +556,17 @@ def test_rapidpro_post_submission(self): self.assertContains(response, 'Successful submission', status_code=201) self.assertTrue(response.has_header('Date')) self.assertEqual(response['Location'], 'http://testserver/submission') - # InstanceID is returned as uuid: + # InstanceID is returned as uuid:. # Retrieving the uuid without the prefix in order to retrieve - # the actual instance + # the Instance object uuid = response.data.get('instanceID').split(':')[1] instance = Instance.objects.get(uuid=uuid) expected_xml = ( - "<{self.xform.survey.name} id=" "'transportation_2011_07_25'>orange" - "") + f"") self.assertEqual(instance.xml, expected_xml) + self.assertEqual(self.xform.survey.name, 'data') def test_legacy_rapidpro_post_submission(self): """ diff --git a/onadata/libs/serializers/data_serializer.py b/onadata/libs/serializers/data_serializer.py index 59e247639c..af3e3d93de 100644 --- a/onadata/libs/serializers/data_serializer.py +++ b/onadata/libs/serializers/data_serializer.py @@ -46,7 +46,8 @@ def create_submission(request, username, data_dict, xform_id): """ Returns validated data object instances """ - xml_string = dict2xform(data_dict, xform_id, root='data') + xml_string = dict2xform( + data_dict, xform_id, username=username) xml_file = BytesIO(xml_string.encode('utf-8')) error, instance = safe_create_instance(username, xml_file, [], None, diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 76b10c3158..fc33adf643 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -112,10 +112,38 @@ def _get_instance(xml, new_uuid, submitted_by, status, xform, checksum): return instance -def dict2xform(jsform, form_id, root=None): +def dict2xform(jsform, form_id, root=None, username=None): + """ + Converts a dictionary containing submission data into an XML + Submission for the appropriate form. + + :param jsform (dict): A python dictionary object containing the submission + data + :param form_id (str or XForm): An XForm object or a string value + representing the forms id_string + :param root (str): An optional string that should be used as the + root nodes name. Defaults to None + :param: username (str): An optional string representing a users + username. Used alongside the `form_id` to + locate the XForm object the user is + trying to submit data too. Defaults to None + :returns: Returns a string containing the Submission XML + :rtype: str + """ if not root: - root = form_id - return u"<{0} id='{1}'>{2}".format( + if username: + if isinstance(form_id, XForm): + root = form_id.survey.name + else: + form = XForm.objects.filter( + id_string__iexact=form_id, + user__username__iexact=username, + deleted_at__isnull=True).first() + root = form.survey.name if form else 'data' + else: + root = 'data' + + return "<{0} id='{1}'>{2}".format( root, form_id, dict2xml(jsform)) From 46dc1a371a6409607733ff8b8e7cb86a814dd2f2 Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Wed, 5 Aug 2020 12:13:22 +0300 Subject: [PATCH 3/3] Add management command to replace form ID root node --- .../commands/replace_form_id_root_node.py | 88 +++++++++++++++++++ .../test_replace_form_id_root_node.py | 67 ++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 onadata/apps/logger/management/commands/replace_form_id_root_node.py create mode 100644 onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py diff --git a/onadata/apps/logger/management/commands/replace_form_id_root_node.py b/onadata/apps/logger/management/commands/replace_form_id_root_node.py new file mode 100644 index 0000000000..9d928ac88a --- /dev/null +++ b/onadata/apps/logger/management/commands/replace_form_id_root_node.py @@ -0,0 +1,88 @@ +""" +Management command used to replace the root node of an Instance when +the root node is the XForm ID +Example usage: + python manage.py replace_form_id_root_node -c -i 1,2,3 +""" +import re +from hashlib import sha256 + +from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone +from django.utils.translation import gettext as _ + +from onadata.apps.logger.models import Instance +from onadata.apps.logger.models.instance import InstanceHistory + + +def replace_form_id_with_correct_root_node( + inst_id: int, root: str = None, commit: bool = False) -> str: + inst: Instance = Instance.objects.get(id=inst_id, deleted_at__isnull=True) + initial_xml = inst.xml + form_id = re.escape(inst.xform.id_string) + if not root: + root = inst.xform.survey.name + + opening_tag_regex = f"<{form_id}" + closing_tag_regex = f"" + edited_xml = re.sub(opening_tag_regex, f'<{root}', initial_xml) + edited_xml = re.sub(closing_tag_regex, f'', edited_xml) + + if commit: + last_edited = timezone.now() + history = InstanceHistory.objects.create( + xml=initial_xml, + checksum=inst.checksum, + xform_instance=inst, + ) + inst.last_edited = last_edited + inst.checksum = sha256(edited_xml.encode('utf-8')).hexdigest() + inst.xml = edited_xml + inst.save() + return f"Modified Instance ID {inst.id} - History object {history.id}" + else: + return edited_xml + + +class Command(BaseCommand): + help = _("Replaces form ID String with 'data' for an instances root node") + + def add_arguments(self, parser): + parser.add_argument( + '--instance-ids', + '-i', + dest='instance_ids', + help='Comma-separated list of instance ids.' + ) + parser.add_argument( + '--commit-changes', + '-c', + action='store_true', + dest='commit', + default=False, + help='Save XML changes' + ) + parser.add_argument( + '--root-node', + '-r', + dest='root', + default=None, + help='Default root node name to replace the form ID with' + ) + + def handle(self, *args, **options): + instance_ids = options.get('instance_ids').split(',') + commit = options.get('commit') + root = options.get('root') + + if not instance_ids: + raise CommandError('No instance id provided.') + + for inst_id in instance_ids: + try: + msg = replace_form_id_with_correct_root_node( + inst_id, root=root, commit=commit) + except Instance.DoesNotExist: + msg = f"Instance with ID {inst_id} does not exist" + + self.stdout.write(msg) diff --git a/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py b/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py new file mode 100644 index 0000000000..d18cd1714d --- /dev/null +++ b/onadata/apps/logger/tests/management/commands/test_replace_form_id_root_node.py @@ -0,0 +1,67 @@ +""" +Module containing the tests for the replace_form_id_root_node +management command +""" +from io import BytesIO + +from django.conf import settings + +from onadata.apps.main.tests.test_base import TestBase +from onadata.apps.logger.import_tools import django_file +from onadata.apps.logger.management.commands.replace_form_id_root_node \ + import replace_form_id_with_correct_root_node +from onadata.libs.utils.logger_tools import create_instance + + +class TestReplaceFormIDRootNodeCommand(TestBase): + """TestReplaceFormIDRootNodeCommand Class""" + + def test_replaces_form_id_root_node(self): + """ + Test that the command correctly replaces the form ID + """ + md = """ + | survey | | | | + | | type | name | label | + | | file | file | File | + | | image | image | Image | + """ + self._create_user_and_login() + xform = self._publish_markdown(md, self.user) + id_string = xform.id_string + + xml_string = f""" + <{id_string} id="{id_string}"> + + uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7 + + Health_2011_03_13.xml_2011-03-15_20-30-28.xml + 1300221157303.jpg + + """ + media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' + '_2011_03_13.xml_2011-03-15_20-30-28/') + image_media = django_file( + path=f'{media_root}1300221157303.jpg', field_name='image', + content_type='image/jpeg') + file_media = django_file( + path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', + field_name='file', content_type='text/xml') + instance = create_instance( + self.user.username, + BytesIO(xml_string.strip().encode('utf-8')), + media_files=[file_media, image_media]) + + # Attempt replacement of root node name + replace_form_id_with_correct_root_node( + inst_id=instance.id, root='data', commit=True) + instance.refresh_from_db() + + expected_xml = f""" + + uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7 + + Health_2011_03_13.xml_2011-03-15_20-30-28.xml + 1300221157303.jpg + """ + self.assertEqual(instance.xml, expected_xml)