Add validate_csv function

- Split submit_csv validation functionality into validate_csv - Update tests - Fix issue where we wouldn't convert xls dates if the first row was null
onaio · Dec 5, 2019 · 1917a0c · 1917a0c
1 parent e485834
commit 1917a0c
Show file tree

Hide file tree

Showing 9 changed files with 278 additions and 246 deletions.
diff --git a/onadata/libs/tests/utils/fixtures/additional.csv b/onadata/libs/tests/utils/fixtures/additional.csv
@@ -4,7 +4,7 @@ Name_2,20,male,NA,2014-09-30,-84.5351 29.0149 100000 1,-84.5351,29.0149,100000,2
 Name_3,21,male,NA,2014-09-29,-84.5351 29.0149 100000 1,-83.5351,30.0149,2456,3,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:01,2014-09-04T12:24:00.000+03:01,2014-09-03,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0b6d4344-6f64-41bc-8bab-a46a5493f9ad,0b6d4344-6f64-41bc-8bab-a46a5493f9ad,2014-09-04T09:24:58,,,bob,,,
 Name_4,22,male,NA,2014-09-28,-84.5351 29.0149 100000 2,-82.5351,31.0149,7653,4,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:02,2014-09-04T12:24:00.000+03:02,2014-09-02,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:15148861-93bc-45b8-ab56-6a9242c5a79d,15148861-93bc-45b8-ab56-6a9242c5a79d,2014-09-04T09:24:59,,,bob,,,
 Name_5,23,male,NA,2014-09-27,-84.5351 29.0149 100000 3,-81.5351,32.0149,245,5,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:03,2014-09-04T12:24:00.000+03:03,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:137e1fb7-81a3-43ae-9039-6f6f599d55a6,137e1fb7-81a3-43ae-9039-6f6f599d55a6,2014-09-04T09:24:32,,,bob,,,
-Name_6,24,male,NA,2014-09-26,-84.5351 29.0149 100000 4,-80.5351,33.0149,65345,6,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:04,2014-09-04T12:24:00.000+03:04,2014-0900,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:fb0af0bf-d476-4136-a51f-13d84f6f9d62,fb0af0bf-d476-4136-a51f-13d84f6f9d62,2014-09-04T09:24:33,,,bob,,,
-Name_7,25,male,NA,2014-09-25,-84.5351 29.0149 100000 5,-79.5351,34.0149,23466,7,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:05,2014-09-04T12:24:00.000+03:05,2014-0901,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:f70bce6b-1785-43fd-8904-e8bb0975838a,f70bce6b-1785-43fd-8904-e8bb0975838a,2014-09-04T09:24:34,,,bob,,,
-Name_8,26,male,NA,2014-09-24,-84.5351 29.0149 100000 6,-78.5351,35.0149,5634562,8,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:06,2014-09-04T12:24:00.000+03:06,2014-0902,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:db78c788-2ea3-4250-ab32-866e946811b6,db78c788-2ea3-4250-ab32-866e946811b6,2014-09-04T09:24:35,,,tori,,,
-Name_9,27,male,NA,2014-09-23,-84.5351 29.0149 100000 7,-77.5351,36.0149,24365,9,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:07,2014-09-04T12:24:00.000+03:07,2014-0903,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,2014-09-04T09:24:36,,,bob,,,
+Name_6,24,male,NA,2014-09-26,-84.5351 29.0149 100000 4,-80.5351,33.0149,65345,6,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:04,2014-09-04T12:24:00.000+03:04,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:fb0af0bf-d476-4136-a51f-13d84f6f9d62,fb0af0bf-d476-4136-a51f-13d84f6f9d62,2014-09-04T09:24:33,,,bob,,,
+Name_7,25,male,NA,2014-09-25,-84.5351 29.0149 100000 5,-79.5351,34.0149,23466,7,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:05,2014-09-04T12:24:00.000+03:05,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:f70bce6b-1785-43fd-8904-e8bb0975838a,f70bce6b-1785-43fd-8904-e8bb0975838a,2014-09-04T09:24:34,,,bob,,,
+Name_8,26,male,NA,2014-09-24,-84.5351 29.0149 100000 6,-78.5351,35.0149,5634562,8,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:06,2014-09-04T12:24:00.000+03:06,2014-09-02,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:db78c788-2ea3-4250-ab32-866e946811b6,db78c788-2ea3-4250-ab32-866e946811b6,2014-09-04T09:24:35,,,tori,,,
+Name_9,27,male,NA,2014-09-23,-84.5351 29.0149 100000 7,-77.5351,36.0149,24365,9,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:07,2014-09-04T12:24:00.000+03:07,2014-09-03,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,2014-09-04T09:24:36,,,bob,,,
diff --git a/...libs/tests/utils/fixtures/bad_integer.csv → ...ta/libs/tests/utils/fixtures/bad_data.csv b/...libs/tests/utils/fixtures/bad_integer.csv → ...ta/libs/tests/utils/fixtures/bad_data.csv
@@ -1,6 +1,6 @@
 name,age,gender,photo,date,location,_location_latitude,_location_longitude,_location_altitude,_location_precision,pizza_fan,pizza_type,favorite_toppings/cheese,favorite_toppings/pepperoni,favorite_toppings/sausauge,favorite_toppings/green_peppers,favorite_toppings/mushrooms,favorite_toppings/anchovies,start_time,end_time,today,imei,phonenumber,meta/instanceID,_uuid,_submission_time,_tags,_notes,_submitted_by,_version,_duration
 Name_2,20.85,male,NA,2014-09-30,-84.5351 29.0149 100000 1,-84.5351,29.0149,100000,2,yes,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:00,2014-09-04T12:24:00.000+03:00,2014-09-04,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:e92dad0d-ee3f-41eb-82d0-4cc0e7f12cb9,e92dad0d-ee3f-41eb-82d0-4cc0e7f12cb9,2014-09-04T09:24:57,,,bob,,
-Name_3,21.53,male,NA,2014-09-29,-84.5351 29.0149 100000 1,-83.5351,30.0149,2456,3,no,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:01,2014-09-04T12:24:00.000+03:01,2014-09-03,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0b6d4344-6f64-41bc-8bab-a46a5493f9ad,0b6d4344-6f64-41bc-8bab-a46a5493f9ad,2014-09-04T09:24:58,,,bob,,
+Name_3,21.53,male,NA,2014-09-29,-84.5351 29.0149 100000 1,-83.5351,30.0149,2456,3,no,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:01,sdsa,2014-0903,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0b6d4344-6f64-41bc-8bab-a46a5493f9ad,0b6d4344-6f64-41bc-8bab-a46a5493f9ad,2014-09-04T09:24:58,,,bob,,
 Name_4,22.32,male,NA,2014-09-28,-84.5351 29.0149 100000 2,-82.5351,31.0149,7653,4,yes,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:02,2014-09-04T12:24:00.000+03:02,2014-09-02,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:15148861-93bc-45b8-ab56-6a9242c5a79d,15148861-93bc-45b8-ab56-6a9242c5a79d,2014-09-04T09:24:59,,,bob,,
 Name_5,23,male,NA,2014-09-27,-84.5351 29.0149 100000 3,-81.5351,32.0149,245,5,no,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:03,2014-09-04T12:24:00.000+03:03,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:137e1fb7-81a3-43ae-9039-6f6f599d55a6,137e1fb7-81a3-43ae-9039-6f6f599d55a6,2014-09-04T09:24:32,,,bob,,
 Name_6,24,male,NA,2014-09-26,-84.5351 29.0149 100000 4,-80.5351,33.0149,65345,6,yes,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:04,2014-09-04T12:24:00.000+03:04,2014-0900,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:fb0af0bf-d476-4136-a51f-13d84f6f9d62,fb0af0bf-d476-4136-a51f-13d84f6f9d62,2014-09-04T09:24:33,,,bob,,

diff --git a/onadata/libs/tests/utils/fixtures/bad_date.csv b/onadata/libs/tests/utils/fixtures/bad_date.csv
diff --git a/onadata/libs/tests/utils/fixtures/bad_datetime.csv b/onadata/libs/tests/utils/fixtures/bad_datetime.csv
diff --git a/onadata/libs/tests/utils/fixtures/bad_decimal.csv b/onadata/libs/tests/utils/fixtures/bad_decimal.csv
diff --git a/onadata/libs/tests/utils/fixtures/bad_decimal.xlsx b/onadata/libs/tests/utils/fixtures/bad_decimal.xlsx
diff --git a/onadata/libs/tests/utils/fixtures/good.xls b/onadata/libs/tests/utils/fixtures/good.xls
diff --git a/onadata/libs/tests/utils/test_csv_import.py b/onadata/libs/tests/utils/test_csv_import.py
@@ -413,33 +413,20 @@ def test_enforces_data_type(self):
         self._publish_xls_file(xls_file_path)
         self.xform = XForm.objects.last()
 
-        bad_integer_csv = open(
-            os.path.join(self.fixtures_dir, 'bad_integer.csv'),
+        bad_data = open(
+            os.path.join(self.fixtures_dir, 'bad_data.csv'),
             'rb')
         result = csv_import.submit_csv(self.user.username, self.xform,
-                                       bad_integer_csv)
+                                       bad_data)
+
+        expected_error = (
+            "Invalid CSV data imported in row(s): {1: ['Unknown integer "
+            "format(s): 20.85'], 2: ['Unknown date format(s): 2014-0903', "
+            "'Unknown datetime format(s): sdsa', 'Unknown integer format(s): "
+            "21.53'], 3: ['Unknown integer format(s): 22.32'], 5: ['Unknown "
+            "date format(s): 2014-0900'], 6: ['Unknown date format(s): "
+            "2014-0901'], 7: ['Unknown date format(s): 2014-0902'], 8: "
+            "['Unknown date format(s): 2014-0903']}")
         self.assertEqual(
             result.get('error'),
-            'Unknown integer format(s): 20.85')
-
-        # Test datetime constraint is enforced
-        bad_date_csv = open(
-            os.path.join(self.fixtures_dir, 'bad_datetime.csv'), 'rb')
-        result = csv_import.submit_csv(
-            self.user.username, self.xform, bad_date_csv)
-        self.assertEqual(
-            result.get('error'),
-            'Unknown datetime format(s): 2931093293232')
-
-        # Test decimal constraint is enforced
-        xls_file_path = os.path.join(self.fixtures_dir, 'bad_decimal.xlsx')
-        self._publish_xls_file(xls_file_path)
-        self.xform = XForm.objects.last()
-        bad_decimal_csv = open(
-            os.path.join(self.fixtures_dir, 'bad_decimal.csv'),
-            'rb')
-        result = csv_import.submit_csv(self.user.username, self.xform,
-                                       bad_decimal_csv)
-        self.assertEqual(
-            result.get('error'),
-            'Unknown decimal format(s): sdsa')
+            expected_error)