Support query by date_modified field (#2009)

* Test that it's possible to filter by date_modified * Add "_date_modified" into Instance JSON * Add 'created' attribute to Data list XML representation * Format files * Add '_date_modified' to the ADDITIONAL_COLUMNS list * Add `DATE_MODIFIED` into the METADATA_FIELDS list * Update tests * Check if date_modified is null * Format files * Modify ordering of additional labels * Set '_date_modified' to the `date_created` value if not set on Instance * Update tests * Format files * Update test fixtures - Add '_date_modified' header * Update tests * Add '_date_modified' to the XForm headers list * Format date_created value before assigning to _date_modified field * Update tests Retrieve _date_modified value from the actual CSV * Update documentation Add the _date_modified field to the sample data for the Data endpoint * Set date_modified to date_created if null * Trigger CI tests
onaio · Feb 16, 2021 · ca1838a · ca1838a
1 parent d33c751
commit ca1838a
Show file tree

Hide file tree

Showing 17 changed files with 182 additions and 107 deletions.
diff --git a/docs/data.rst b/docs/data.rst
@@ -156,6 +156,7 @@ Response
                 "formhub/uuid": "46ea15e2b8134624a47e2c4b77eef0d4",
                 "kind": "monthly",
                 "_submission_time": "2013-01-03T02:27:19",
+                "_date_modified": "2013-01-03T02:29:20",
                 "_submitted_by": "onaio",
                 "required": "yes",
                 "_attachments": [],

diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py
@@ -868,6 +868,31 @@ def test_filter_by_submission_time_and_submitted_by(self):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(len(response.data), 2)
 
+    def test_filter_by_date_modified(self):
+        self._make_submissions()
+        view = DataViewSet.as_view({'get': 'list'})
+        request = self.factory.get('/', **self.extra)
+        formid = self.xform.pk
+        instance = self.xform.instances.all().order_by('pk')[0]
+        response = view(request, pk=formid)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data), 4)
+
+        instance = self.xform.instances.all().order_by('-date_created')[0]
+        date_modified = instance.date_modified.strftime(MONGO_STRFTIME)
+
+        query_str = ('{"_date_modified": {"$gte": "%s"},'
+                     ' "_submitted_by": "%s"}' % (date_modified, 'bob'))
+        data = {
+            'query': query_str
+        }
+        request = self.factory.get('/', data=data, **self.extra)
+        response = view(request, pk=formid)
+        self.assertEqual(response.status_code, 200)
+        expected_count = self.xform.instances.filter(
+            json___date_modified__gte=date_modified).count()
+        self.assertEqual(len(response.data), expected_count)
+
     def test_filter_by_submission_time_and_submitted_by_with_data_arg(self):
         self._make_submissions()
         view = DataViewSet.as_view({'get': 'list'})
@@ -2533,7 +2558,7 @@ def test_data_list_xml_format(self):
         server_time = ET.fromstring(returned_xml).attrib.get('serverTime')
         expected_xml = (
             f'<?xml version="1.0" encoding="utf-8"?>\n<submission-batch serverTime="{server_time}">'  # noqa
-            f'<submission-item formVersion="{instance.version}" lastModified="{instance.date_modified.isoformat()}" objectID="{instance.id}">'  # noqa
+            f'<submission-item created="{instance.date_created.isoformat()}" formVersion="{instance.version}" lastModified="{instance.date_modified.isoformat()}" objectID="{instance.id}">'  # noqa
             '<tutorial id="tutorial"><name>Larry\n        Again</name><age>23</age><picture>1333604907194.jpg</picture>'  # noqa
             '<has_children>0</has_children><gps>-1.2836198 36.8795437 0.0 1044.0</gps><web_browsers>firefox chrome safari'  # noqa
             '</web_browsers><meta><instanceID>uuid:729f173c688e482486a48661700455ff</instanceID></meta></tutorial>'  # noqa

diff --git a/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_merged_xform_viewset.py
@@ -428,8 +428,9 @@ def test_md_csv_export(self):
         headers = next(csv_reader)
         self.assertEqual(headers, [
             'fruit', 'meta/instanceID', '_id', '_uuid', '_submission_time',
-            '_tags', '_notes', '_version', '_duration', '_submitted_by',
-            '_total_media', '_media_count', '_media_all_received'])
+            '_date_modified', '_tags', '_notes', '_version', '_duration',
+            '_submitted_by', '_total_media', '_media_count',
+            '_media_all_received'])
         row1 = next(csv_reader)
         self.assertEqual(row1[0], 'orange')
         row2 = next(csv_reader)

diff --git a/onadata/apps/api/tests/viewsets/test_open_data_viewset.py b/onadata/apps/api/tests/viewsets/test_open_data_viewset.py
@@ -387,7 +387,7 @@ def test_tableau_data_fetch(self):  # pylint: disable=invalid-name
             'web_browsers_chrome',
             'web_browsers_firefox',
             'web_browsers_ie',
-            'web_browsers_safari'
+            'web_browsers_safari',
         ]
 
         self.view = OpenDataViewSet.as_view({

diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py
@@ -3405,31 +3405,36 @@ def test_csv_export_with_win_excel_utf8(self):
             request = self.factory.get('/', data=data, **self.extra)
             response = view(request, pk=self.xform.pk, format='csv')
             self.assertEqual(response.status_code, 200)
-            data_id = self.xform.instances.first().pk
+            instance = self.xform.instances.first()
+            data_id, date_modified = (
+                    instance.pk,
+                    instance.date_modified.strftime(MONGO_STRFTIME))
 
             content = get_response_content(response)
 
             expected_content_py2 = (
                 '\ufeffage,\ufeffname,\ufeffmeta/instanceID,\ufeff_id,'
-                '\ufeff_uuid,\ufeff_submission_time,\ufeff_tags,\ufeff_notes,'
-                '\ufeff_version,\ufeff_duration,\ufeff_submitted_by,'
-                '\ufeff_total_media,\ufeff_media_count,'
+                '\ufeff_uuid,\ufeff_submission_time,\ufeff_date_modified,'
+                '\ufeff_tags,\ufeff_notes,\ufeff_version,\ufeff_duration,'
+                '\ufeff_submitted_by,\ufeff_total_media,\ufeff_media_count,'
                 '\ufeff_media_all_received\n'
                 '\ufeff#age,,,,,,,,,,,,,\n29,'
                 '\ufeffLionel Messi,'
                 '\ufeffuuid:74ee8b73-48aa-4ced-9072-862f93d49c16,%s,'
                 '\ufeff74ee8b73-48aa-4ced-9072-862f93d49c16,'
-                '\ufeff2013-02-18T15:54:01,\ufeff,\ufeff,\ufeff201604121155,'
-                '\ufeff,\ufeffbob,0,0,True\n' % data_id)
+                '\ufeff2013-02-18T15:54:01,%s,\ufeff,\ufeff,\ufeff'
+                '201604121155,\ufeff,\ufeffbob,0,0,True\n'
+                % (data_id, date_modified))
+
             expected_content_py3 = (
                 '\ufeffage,name,meta/instanceID,_id,_uuid,_submission_time,'
-                '_tags,_notes,_version,_duration,_submitted_by,_total_media,'
-                '_media_count,'
-                '_media_all_received\n\ufeff#age,,,,,,,,,,,,,\n'
-                '\ufeff29,Lionel Messi,'
-                'uuid:74ee8b73-48aa-4ced-9072-862f93d49c16,%s,'
-                '74ee8b73-48aa-4ced-9072-862f93d49c16,2013-02-18T15:54:01,,,'
-                '201604121155,,bob,0,0,True\n' % data_id)
+                '_date_modified,_tags,_notes,_version,_duration,_submitted_by,'
+                '_total_media,_media_count,_media_all_received\n\ufeff#age'
+                ',,,,,,,,,,,,,,\n\ufeff'
+                '29,Lionel Messi,uuid:74ee8b73-48aa-4ced-9072-862f93d49c16,'
+                '%s,74ee8b73-48aa-4ced-9072-862f93d49c16,2013-02-18T15:54:01,'
+                '%s,,,201604121155,,bob,0,0,True\n' % (data_id, date_modified)
+            )
             self.assertIn(content, [expected_content_py2,
                                     expected_content_py3])
             headers = dict(response.items())
@@ -3446,13 +3451,13 @@ def test_csv_export_with_win_excel_utf8(self):
 
             content = get_response_content(response)
             expected_content = (
-                'age,name,meta/instanceID,_id,_uuid,_submission_time,_tags,'
-                '_notes,_version,_duration,_submitted_by,_total_media,'
-                '_media_count,_media_all_received\n'
-                '#age,,,,,,,,,,,,,\n'
+                'age,name,meta/instanceID,_id,_uuid,_submission_time,'
+                '_date_modified,_tags,_notes,_version,_duration,_submi'
+                'tted_by,_total_media,_media_count,_media_all_received\n'
+                '#age,,,,,,,,,,,,,,\n'
                 '29,Lionel Messi,uuid:74ee8b73-48aa-4ced-9072-862f93d49c16'
                 ',%s,74ee8b73-48aa-4ced-9072-862f93d49c16,2013-02-18T15:54:01,'
-                ',,201604121155,,bob,0,0,True\n' % data_id
+                '%s,,,201604121155,,bob,0,0,True\n' % (data_id, date_modified)
             )
 
             self.assertEqual(expected_content, content)
@@ -3479,7 +3484,9 @@ def test_csv_export_with_and_without_include_hxl(self):
                     "hxl_test", "hxl_example.xml"),
                 forced_submission_time=_submission_time)
             self.assertTrue(self.xform.has_hxl_support)
-            data_id = self.xform.instances.first().pk
+            instance = self.xform.instances.first()
+            data_id, date_modified = (
+                instance.pk, instance.date_modified.strftime(MONGO_STRFTIME))
 
             view = XFormViewSet.as_view({
                 'get': 'retrieve'
@@ -3498,12 +3505,12 @@ def test_csv_export_with_and_without_include_hxl(self):
 
             content = get_response_content(response)
             expected_content = (
-                'age,name,meta/instanceID,_id,_uuid,_submission_time,_tags,'
-                '_notes,_version,_duration,_submitted_by,_total_media,'
-                '_media_count,_media_all_received\n'
+                'age,name,meta/instanceID,_id,_uuid,_submission_time,'
+                '_date_modified,_tags,_notes,_version,_duration,_submitted_by,'
+                '_total_media,_media_count,_media_all_received\n'
                 '29,Lionel Messi,uuid:74ee8b73-48aa-4ced-9072-862f93d49c16,'
-                '%s,74ee8b73-48aa-4ced-9072-862f93d49c16,2013-02-18T15:54:01,,'
-                ',201604121155,,bob,0,0,True\n' % data_id
+                '%s,74ee8b73-48aa-4ced-9072-862f93d49c16,2013-02-18T15:54:01,'
+                '%s,,,201604121155,,bob,0,0,True\n' % (data_id, date_modified)
             )
             self.assertEqual(expected_content, content)
             headers = dict(response.items())
@@ -3519,13 +3526,14 @@ def test_csv_export_with_and_without_include_hxl(self):
 
             content = get_response_content(response)
             expected_content = (
-                'age,name,meta/instanceID,_id,_uuid,_submission_time,_tags,'
-                '_notes,_version,_duration,_submitted_by,_total_media,'
-                '_media_count,_media_all_received\n'
-                '#age,,,,,,,,,,,,,\n'
+                'age,name,meta/instanceID,_id,_uuid,_submission_time,'
+                '_date_modified,_tags,_notes,_version,_duration,'
+                '_submitted_by,_total_media,_media_count,'
+                '_media_all_received\n'
+                '#age,,,,,,,,,,,,,,\n'
                 '29,Lionel Messi,uuid:74ee8b73-48aa-4ced-9072-862f93d49c16,'
                 '%s,74ee8b73-48aa-4ced-9072-862f93d49c16,2013-02-18T15:54:01'
-                ',,,201604121155,,bob,0,0,True\n' % data_id
+                ',%s,,,201604121155,,bob,0,0,True\n' % (data_id, date_modified)
             )
             self.assertEqual(expected_content, content)
             headers = dict(response.items())

diff --git a/onadata/apps/logger/migrations/0062_auto_20210202_0248.py b/onadata/apps/logger/migrations/0062_auto_20210202_0248.py
@@ -0,0 +1,25 @@
+# pylint: skip-file
+# Generated by Django 2.2.16 on 2021-02-02 07:48
+
+from django.db import migrations
+from onadata.apps.logger.models.instance import Instance
+
+
+def regenerate_instance_json(apps, schema_editor):
+    """
+    Regenerate Instance JSON
+    """
+    for inst in Instance.objects.filter(deleted_at__isnull=True):
+        inst.json = inst.get_full_dict(load_existing=False)
+        inst.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('logger', '0061_auto_20200713_0814'),
+    ]
+
+    operations = [
+        migrations.RunPython(regenerate_instance_json)
+    ]
diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py
@@ -34,16 +34,13 @@
     DATAVIEW_COUNT, IS_ORG, PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE,
     XFORM_COUNT, XFORM_DATA_VERSIONS, XFORM_SUBMISSION_COUNT_FOR_DAY,
     XFORM_SUBMISSION_COUNT_FOR_DAY_DATE, safe_delete)
-from onadata.libs.utils.common_tags import (ATTACHMENTS, BAMBOO_DATASET_ID,
-                                            DELETEDAT, DURATION, EDITED, END,
-                                            GEOLOCATION, ID, LAST_EDITED,
-                                            MEDIA_ALL_RECEIVED, MEDIA_COUNT,
-                                            MONGO_STRFTIME, NOTES,
-                                            REVIEW_STATUS, START, STATUS,
-                                            SUBMISSION_TIME, SUBMITTED_BY,
-                                            TAGS, TOTAL_MEDIA, UUID, VERSION,
-                                            XFORM_ID, XFORM_ID_STRING,
-                                            REVIEW_COMMENT)
+from onadata.libs.utils.common_tags import (
+    ATTACHMENTS, BAMBOO_DATASET_ID, DATE_MODIFIED,
+    DELETEDAT, DURATION, EDITED, END, GEOLOCATION, ID, LAST_EDITED,
+    MEDIA_ALL_RECEIVED, MEDIA_COUNT, MONGO_STRFTIME, NOTES,
+    REVIEW_STATUS, START, STATUS, SUBMISSION_TIME, SUBMITTED_BY,
+    TAGS, TOTAL_MEDIA, UUID, VERSION, XFORM_ID, XFORM_ID_STRING,
+    REVIEW_COMMENT)
 from onadata.libs.utils.dict_tools import get_values_matching_key
 from onadata.libs.utils.model_tools import set_uuid
 from onadata.libs.utils.timing import calculate_duration
@@ -388,6 +385,12 @@ def get_full_dict(self, load_existing=True):
             if not self.date_created:
                 self.date_created = submission_time()
 
+            if not self.date_modified:
+                self.date_modified = self.date_created
+
+            doc[DATE_MODIFIED] = self.date_modified.strftime(
+                    MONGO_STRFTIME)
+
             doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
 
             doc[TOTAL_MEDIA] = self.total_media

diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py
@@ -46,7 +46,8 @@
                                             SUBMITTED_BY, TAGS, TOTAL_MEDIA,
                                             UUID, VERSION, REVIEW_STATUS,
                                             REVIEW_COMMENT,
-                                            MULTIPLE_SELECT_TYPE)
+                                            MULTIPLE_SELECT_TYPE,
+                                            DATE_MODIFIED)
 from onadata.libs.utils.model_tools import queryset_iterator
 from onadata.libs.utils.mongo import _encode_for_mongo
 
@@ -489,9 +490,10 @@ def shorten(xpath):
             shorten(xpath) for xpath in self.xpaths(
                 repeat_iterations=repeat_iterations)]
         header_list += [
-            ID, UUID, SUBMISSION_TIME, TAGS, NOTES, REVIEW_STATUS,
-            REVIEW_COMMENT, VERSION, DURATION, SUBMITTED_BY, TOTAL_MEDIA,
-            MEDIA_COUNT, MEDIA_ALL_RECEIVED
+            ID, UUID, SUBMISSION_TIME, DATE_MODIFIED, TAGS, NOTES,
+            REVIEW_STATUS, REVIEW_COMMENT, VERSION, DURATION,
+            SUBMITTED_BY, TOTAL_MEDIA, MEDIA_COUNT,
+            MEDIA_ALL_RECEIVED
         ]
         if include_additional_headers:
             header_list += self._additional_headers()

diff --git a/onadata/apps/main/tests/fixtures/transportation/headers.json b/onadata/apps/main/tests/fixtures/transportation/headers.json
@@ -24,6 +24,7 @@
     "_id",
     "_uuid",
     "_submission_time",
+    "_date_modified",
     "_tags",
     "_notes",
     "_review_status",

diff --git a/onadata/apps/main/tests/fixtures/transportation/headers_csv.json b/onadata/apps/main/tests/fixtures/transportation/headers_csv.json
@@ -24,6 +24,7 @@
     "_id",
     "_uuid",
     "_submission_time",
+    "_date_modified",
     "_tags",
     "_notes",
     "_version",

diff --git a/onadata/apps/main/tests/test_csv_export.py b/onadata/apps/main/tests/test_csv_export.py
@@ -166,6 +166,7 @@ def test_csv_repeat_with_note(self):
             expected_headers = [
                 'chnum', 'chrepeat[1]/chname', 'chrepeat[2]/chname',
                 'meta/instanceID', '_id', '_uuid', '_submission_time',
-                '_tags', '_notes', '_version', '_duration', '_submitted_by',
-                '_total_media', '_media_count', '_media_all_received']
+                '_date_modified', '_tags', '_notes', '_version',
+                '_duration', '_submitted_by', '_total_media',
+                '_media_count', '_media_all_received']
             self.assertEqual(sorted(expected_headers), sorted(actual_headers))
diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py
@@ -437,7 +437,8 @@ def _check_csv_export_second_pass(self):
         ]
 
         dd = DataDictionary.objects.get(pk=self.xform.pk)
-        additional_headers = dd._additional_headers() + ['_id']
+        additional_headers = dd._additional_headers() + [
+            '_id', '_date_modified']
         for row, expected_dict in zip(actual_csv, data):
             test_dict = {}
             d = dict(zip(headers, row))

diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py
@@ -23,7 +23,7 @@
     GEOLOCATION, SUBMISSION_TIME, MONGO_STRFTIME, BAMBOO_DATASET_ID, \
     DELETEDAT, TAGS, NOTES, SUBMITTED_BY, VERSION, DURATION, EDITED, \
     MEDIA_COUNT, TOTAL_MEDIA, MEDIA_ALL_RECEIVED, XFORM_ID, REVIEW_STATUS, \
-    REVIEW_COMMENT
+    REVIEW_COMMENT, DATE_MODIFIED
 from onadata.libs.utils.model_tools import queryset_iterator
 from onadata.libs.utils.mongo import _is_invalid_for_mongo
 
@@ -285,6 +285,8 @@ def to_dict_for_mongo(self):
             GEOLOCATION: [self.lat, self.lng],
             SUBMISSION_TIME: self.instance.date_created.strftime(
                 MONGO_STRFTIME),
+            DATE_MODIFIED: self.instance.date_modified.strftime(
+                MONGO_STRFTIME),
             TAGS: list(self.instance.tags.names()),
             NOTES: self.get_notes(),
             SUBMITTED_BY: self.instance.user.username

diff --git a/onadata/libs/serializers/data_serializer.py b/onadata/libs/serializers/data_serializer.py
@@ -120,6 +120,7 @@ def to_representation(self, instance):
         instance_attributes = {
             '@formVersion': instance.version,
             '@lastModified': instance.date_modified.isoformat(),
+            '@created': instance.date_created.isoformat(),
             '@objectID': str(instance.id)
         }
         ret.update(instance_attributes)