From 1917a0c06487c469c1bb82111ca59e31bfb8d5cb Mon Sep 17 00:00:00 2001 From: Davis Raymond Muro Date: Thu, 5 Dec 2019 10:43:47 +0300 Subject: [PATCH] Add validate_csv function - Split submit_csv validation functionality into validate_csv - Update tests - Fix issue where we wouldn't convert xls dates if the first row was null --- .../libs/tests/utils/fixtures/additional.csv | 8 +- .../{bad_integer.csv => bad_data.csv} | 2 +- .../libs/tests/utils/fixtures/bad_date.csv | 10 - .../tests/utils/fixtures/bad_datetime.csv | 9 - .../libs/tests/utils/fixtures/bad_decimal.csv | 2 - .../tests/utils/fixtures/bad_decimal.xlsx | Bin 5846 -> 0 bytes onadata/libs/tests/utils/fixtures/good.xls | Bin 16560 -> 16644 bytes onadata/libs/tests/utils/test_csv_import.py | 39 +- onadata/libs/utils/csv_import.py | 454 ++++++++++-------- 9 files changed, 278 insertions(+), 246 deletions(-) rename onadata/libs/tests/utils/fixtures/{bad_integer.csv => bad_data.csv} (92%) delete mode 100644 onadata/libs/tests/utils/fixtures/bad_date.csv delete mode 100644 onadata/libs/tests/utils/fixtures/bad_datetime.csv delete mode 100644 onadata/libs/tests/utils/fixtures/bad_decimal.csv delete mode 100644 onadata/libs/tests/utils/fixtures/bad_decimal.xlsx diff --git a/onadata/libs/tests/utils/fixtures/additional.csv b/onadata/libs/tests/utils/fixtures/additional.csv index e920f9f1d7..e8969a95d1 100644 --- a/onadata/libs/tests/utils/fixtures/additional.csv +++ b/onadata/libs/tests/utils/fixtures/additional.csv @@ -4,7 +4,7 @@ Name_2,20,male,NA,2014-09-30,-84.5351 29.0149 100000 1,-84.5351,29.0149,100000,2 Name_3,21,male,NA,2014-09-29,-84.5351 29.0149 100000 1,-83.5351,30.0149,2456,3,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:01,2014-09-04T12:24:00.000+03:01,2014-09-03,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0b6d4344-6f64-41bc-8bab-a46a5493f9ad,0b6d4344-6f64-41bc-8bab-a46a5493f9ad,2014-09-04T09:24:58,,,bob,,, Name_4,22,male,NA,2014-09-28,-84.5351 29.0149 100000 2,-82.5351,31.0149,7653,4,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:02,2014-09-04T12:24:00.000+03:02,2014-09-02,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:15148861-93bc-45b8-ab56-6a9242c5a79d,15148861-93bc-45b8-ab56-6a9242c5a79d,2014-09-04T09:24:59,,,bob,,, Name_5,23,male,NA,2014-09-27,-84.5351 29.0149 100000 3,-81.5351,32.0149,245,5,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:03,2014-09-04T12:24:00.000+03:03,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:137e1fb7-81a3-43ae-9039-6f6f599d55a6,137e1fb7-81a3-43ae-9039-6f6f599d55a6,2014-09-04T09:24:32,,,bob,,, -Name_6,24,male,NA,2014-09-26,-84.5351 29.0149 100000 4,-80.5351,33.0149,65345,6,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:04,2014-09-04T12:24:00.000+03:04,2014-0900,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:fb0af0bf-d476-4136-a51f-13d84f6f9d62,fb0af0bf-d476-4136-a51f-13d84f6f9d62,2014-09-04T09:24:33,,,bob,,, -Name_7,25,male,NA,2014-09-25,-84.5351 29.0149 100000 5,-79.5351,34.0149,23466,7,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:05,2014-09-04T12:24:00.000+03:05,2014-0901,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:f70bce6b-1785-43fd-8904-e8bb0975838a,f70bce6b-1785-43fd-8904-e8bb0975838a,2014-09-04T09:24:34,,,bob,,, -Name_8,26,male,NA,2014-09-24,-84.5351 29.0149 100000 6,-78.5351,35.0149,5634562,8,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:06,2014-09-04T12:24:00.000+03:06,2014-0902,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:db78c788-2ea3-4250-ab32-866e946811b6,db78c788-2ea3-4250-ab32-866e946811b6,2014-09-04T09:24:35,,,tori,,, -Name_9,27,male,NA,2014-09-23,-84.5351 29.0149 100000 7,-77.5351,36.0149,24365,9,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:07,2014-09-04T12:24:00.000+03:07,2014-0903,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,2014-09-04T09:24:36,,,bob,,, +Name_6,24,male,NA,2014-09-26,-84.5351 29.0149 100000 4,-80.5351,33.0149,65345,6,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:04,2014-09-04T12:24:00.000+03:04,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:fb0af0bf-d476-4136-a51f-13d84f6f9d62,fb0af0bf-d476-4136-a51f-13d84f6f9d62,2014-09-04T09:24:33,,,bob,,, +Name_7,25,male,NA,2014-09-25,-84.5351 29.0149 100000 5,-79.5351,34.0149,23466,7,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:05,2014-09-04T12:24:00.000+03:05,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:f70bce6b-1785-43fd-8904-e8bb0975838a,f70bce6b-1785-43fd-8904-e8bb0975838a,2014-09-04T09:24:34,,,bob,,, +Name_8,26,male,NA,2014-09-24,-84.5351 29.0149 100000 6,-78.5351,35.0149,5634562,8,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:06,2014-09-04T12:24:00.000+03:06,2014-09-02,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:db78c788-2ea3-4250-ab32-866e946811b6,db78c788-2ea3-4250-ab32-866e946811b6,2014-09-04T09:24:35,,,tori,,, +Name_9,27,male,NA,2014-09-23,-84.5351 29.0149 100000 7,-77.5351,36.0149,24365,9,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:07,2014-09-04T12:24:00.000+03:07,2014-09-03,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,2014-09-04T09:24:36,,,bob,,, diff --git a/onadata/libs/tests/utils/fixtures/bad_integer.csv b/onadata/libs/tests/utils/fixtures/bad_data.csv similarity index 92% rename from onadata/libs/tests/utils/fixtures/bad_integer.csv rename to onadata/libs/tests/utils/fixtures/bad_data.csv index 28a6710a06..5994dfb5ac 100644 --- a/onadata/libs/tests/utils/fixtures/bad_integer.csv +++ b/onadata/libs/tests/utils/fixtures/bad_data.csv @@ -1,6 +1,6 @@ name,age,gender,photo,date,location,_location_latitude,_location_longitude,_location_altitude,_location_precision,pizza_fan,pizza_type,favorite_toppings/cheese,favorite_toppings/pepperoni,favorite_toppings/sausauge,favorite_toppings/green_peppers,favorite_toppings/mushrooms,favorite_toppings/anchovies,start_time,end_time,today,imei,phonenumber,meta/instanceID,_uuid,_submission_time,_tags,_notes,_submitted_by,_version,_duration Name_2,20.85,male,NA,2014-09-30,-84.5351 29.0149 100000 1,-84.5351,29.0149,100000,2,yes,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:00,2014-09-04T12:24:00.000+03:00,2014-09-04,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:e92dad0d-ee3f-41eb-82d0-4cc0e7f12cb9,e92dad0d-ee3f-41eb-82d0-4cc0e7f12cb9,2014-09-04T09:24:57,,,bob,, -Name_3,21.53,male,NA,2014-09-29,-84.5351 29.0149 100000 1,-83.5351,30.0149,2456,3,no,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:01,2014-09-04T12:24:00.000+03:01,2014-09-03,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0b6d4344-6f64-41bc-8bab-a46a5493f9ad,0b6d4344-6f64-41bc-8bab-a46a5493f9ad,2014-09-04T09:24:58,,,bob,, +Name_3,21.53,male,NA,2014-09-29,-84.5351 29.0149 100000 1,-83.5351,30.0149,2456,3,no,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:01,sdsa,2014-0903,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0b6d4344-6f64-41bc-8bab-a46a5493f9ad,0b6d4344-6f64-41bc-8bab-a46a5493f9ad,2014-09-04T09:24:58,,,bob,, Name_4,22.32,male,NA,2014-09-28,-84.5351 29.0149 100000 2,-82.5351,31.0149,7653,4,yes,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:02,2014-09-04T12:24:00.000+03:02,2014-09-02,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:15148861-93bc-45b8-ab56-6a9242c5a79d,15148861-93bc-45b8-ab56-6a9242c5a79d,2014-09-04T09:24:59,,,bob,, Name_5,23,male,NA,2014-09-27,-84.5351 29.0149 100000 3,-81.5351,32.0149,245,5,no,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:03,2014-09-04T12:24:00.000+03:03,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:137e1fb7-81a3-43ae-9039-6f6f599d55a6,137e1fb7-81a3-43ae-9039-6f6f599d55a6,2014-09-04T09:24:32,,,bob,, Name_6,24,male,NA,2014-09-26,-84.5351 29.0149 100000 4,-80.5351,33.0149,65345,6,yes,chitown,TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,2014-09-04T12:19:31.000+03:04,2014-09-04T12:24:00.000+03:04,2014-0900,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:fb0af0bf-d476-4136-a51f-13d84f6f9d62,fb0af0bf-d476-4136-a51f-13d84f6f9d62,2014-09-04T09:24:33,,,bob,, diff --git a/onadata/libs/tests/utils/fixtures/bad_date.csv b/onadata/libs/tests/utils/fixtures/bad_date.csv deleted file mode 100644 index 47eb39fffc..0000000000 --- a/onadata/libs/tests/utils/fixtures/bad_date.csv +++ /dev/null @@ -1,10 +0,0 @@ -name,age,gender,photo,date,location,_location_latitude,_location_longitude,_location_altitude,_location_precision,pizza_fan,pizza_type,favorite_toppings/cheese,favorite_toppings/pepperoni,favorite_toppings/sausauge,favorite_toppings/green_peppers,favorite_toppings/mushrooms,favorite_toppings/anchovies,start_time,end_time,today,imei,phonenumber,meta/instanceID,_uuid,_submission_time,_tags,_notes,_submitted_by,_version,_duration -Name_1,10,male,NA,n/a,83.3595 -32.8601 0 1,83.3595,-32.8601,0,1,no,n/a,n/a,n/a,n/a,n/a,n/a,n/a,2014-09-04T15:06:01.000+03:00,2014-09-04T15:07:17.000+03:00,2014-09-04,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:685dd371-4831-4fdc-a205-f285337dd98d,685dd371-4831-4fdc-a205-f285337dd98d,2014-09-04T12:08:04,,,bob,, -Name_2,20,male,NA,2014-09-30,-84.5351 29.0149 100000 1,-84.5351,29.0149,100000,2,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:00,2014-09-04T12:24:00.000+03:00,2014-09-04,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:e92dad0d-ee3f-41eb-82d0-4cc0e7f12cb9,e92dad0d-ee3f-41eb-82d0-4cc0e7f12cb9,2014-09-04T09:24:57,,,bob,, -Name_3,21,male,NA,2014-09-29,-84.5351 29.0149 100000 1,-83.5351,30.0149,2456,3,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:01,2014-09-04T12:24:00.000+03:01,2014-09-03,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0b6d4344-6f64-41bc-8bab-a46a5493f9ad,0b6d4344-6f64-41bc-8bab-a46a5493f9ad,2014-09-04T09:24:58,,,bob,, -Name_4,22,male,NA,2014-09-28,-84.5351 29.0149 100000 2,-82.5351,31.0149,7653,4,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:02,2014-09-04T12:24:00.000+03:02,2014-09-02,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:15148861-93bc-45b8-ab56-6a9242c5a79d,15148861-93bc-45b8-ab56-6a9242c5a79d,2014-09-04T09:24:59,,,bob,, -Name_5,23,male,NA,2014-09-27,-84.5351 29.0149 100000 3,-81.5351,32.0149,245,5,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:03,2014-09-04T12:24:00.000+03:03,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:137e1fb7-81a3-43ae-9039-6f6f599d55a6,137e1fb7-81a3-43ae-9039-6f6f599d55a6,2014-09-04T09:24:32,,,bob,, -Name_6,24,male,NA,2014-09-26,-84.5351 29.0149 100000 4,-80.5351,33.0149,65345,6,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:04,2014-09-04T12:24:00.000+03:04,2014-0900,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:fb0af0bf-d476-4136-a51f-13d84f6f9d62,fb0af0bf-d476-4136-a51f-13d84f6f9d62,2014-09-04T09:24:33,,,bob,, -Name_7,25,male,NA,2014-09-25,-84.5351 29.0149 100000 5,-79.5351,34.0149,23466,7,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:05,2014-09-04T12:24:00.000+03:05,2014-0901,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:f70bce6b-1785-43fd-8904-e8bb0975838a,f70bce6b-1785-43fd-8904-e8bb0975838a,2014-09-04T09:24:34,,,bob,, -Name_8,26,male,NA,2014-09-24,-84.5351 29.0149 100000 6,-78.5351,35.0149,5634562,8,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:06,2014-09-04T12:24:00.000+03:06,2014-0902,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:db78c788-2ea3-4250-ab32-866e946811b6,db78c788-2ea3-4250-ab32-866e946811b6,2014-09-04T09:24:35,,,tori,, -Name_9,27,male,NA,2014-09-23,-84.5351 29.0149 100000 7,-77.5351,36.0149,24365,9,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:07,2014-09-04T12:24:00.000+03:07,2014-0903,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,2014-09-04T09:24:36,,,bob,, diff --git a/onadata/libs/tests/utils/fixtures/bad_datetime.csv b/onadata/libs/tests/utils/fixtures/bad_datetime.csv deleted file mode 100644 index 2f058d9a9f..0000000000 --- a/onadata/libs/tests/utils/fixtures/bad_datetime.csv +++ /dev/null @@ -1,9 +0,0 @@ -name,age,gender,photo,date,location,_location_latitude,_location_longitude,_location_altitude,_location_precision,pizza_fan,pizza_type,favorite_toppings/cheese,favorite_toppings/pepperoni,favorite_toppings/sausauge,favorite_toppings/green_peppers,favorite_toppings/mushrooms,favorite_toppings/anchovies,start_time,end_time,today,imei,phonenumber,meta/instanceID,_uuid,_submission_time,_tags,_notes,_submitted_by,_version,_duration -Name_2,20,male,NA,2014-09-30,-84.5351 29.0149 100000 1,-84.5351,29.0149,100000,2,yes,chitown,True,True,True,True,True,True,sdssads,2014-09-04T12:24:00.000+03:00,2014-09-04,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:e92dad0d-ee3f-41eb-82d0-4cc0e7f12cb9,e92dad0d-ee3f-41eb-82d0-4cc0e7f12cb9,2014-09-04T09:24:57,,,bob,, -Name_3,21,male,NA,2014-09-29,-84.5351 29.0149 100000 1,-83.5351,30.0149,2456,3,no,chitown,True,True,True,True,True,True,sddsadsd,2014-09-04T12:24:00.000+03:01,2014-09-03,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0b6d4344-6f64-41bc-8bab-a46a5493f9ad,0b6d4344-6f64-41bc-8bab-a46a5493f9ad,2014-09-04T09:24:58,,,bob,, -Name_4,22,male,NA,2014-09-28,-84.5351 29.0149 100000 2,-82.5351,31.0149,7653,4,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:02,2014-09-04T12:24:00.000+03:02,2014-09-02,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:15148861-93bc-45b8-ab56-6a9242c5a79d,15148861-93bc-45b8-ab56-6a9242c5a79d,2014-09-04T09:24:59,,,bob,, -Name_5,23,male,NA,2014-09-27,-84.5351 29.0149 100000 3,-81.5351,32.0149,245,5,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:03,2014-09-04T12:24:00.000+03:03,2014-09-01,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:137e1fb7-81a3-43ae-9039-6f6f599d55a6,137e1fb7-81a3-43ae-9039-6f6f599d55a6,2014-09-04T09:24:32,,,bob,, -Name_6,24,male,NA,2014-09-26,-84.5351 29.0149 100000 4,-80.5351,33.0149,65345,6,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:04,2014-09-04T12:24:00.000+03:04,2014-0900,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:fb0af0bf-d476-4136-a51f-13d84f6f9d62,fb0af0bf-d476-4136-a51f-13d84f6f9d62,2014-09-04T09:24:33,,,bob,, -Name_7,25,male,NA,2014-09-25,-84.5351 29.0149 100000 5,-79.5351,34.0149,23466,7,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:05,2014-09-04T12:24:00.000+03:05,2014-0901,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:f70bce6b-1785-43fd-8904-e8bb0975838a,f70bce6b-1785-43fd-8904-e8bb0975838a,2014-09-04T09:24:34,,,bob,, -Name_8,26,male,NA,2014-09-24,-84.5351 29.0149 100000 6,-78.5351,35.0149,5634562,8,yes,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:06,2014-09-04T12:24:00.000+03:06,2014-0902,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:db78c788-2ea3-4250-ab32-866e946811b6,db78c788-2ea3-4250-ab32-866e946811b6,2014-09-04T09:24:35,,,tori,, -Name_9,27,male,NA,2014-09-23,-84.5351 29.0149 100000 7,-77.5351,36.0149,24365,9,no,chitown,True,True,True,True,True,True,2014-09-04T12:19:31.000+03:07,2014-09-04T12:24:00.000+03:07,2014-0903,enketo.org:2gnoXEilHRGn6V5i,no phonenumber property in enketo,uuid:0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,0e1accb5-1c43-4789-ad2f-b9c663bbbc5d,2014-09-04T09:24:36,,,bob,, diff --git a/onadata/libs/tests/utils/fixtures/bad_decimal.csv b/onadata/libs/tests/utils/fixtures/bad_decimal.csv deleted file mode 100644 index 80e033c837..0000000000 --- a/onadata/libs/tests/utils/fixtures/bad_decimal.csv +++ /dev/null @@ -1,2 +0,0 @@ -Enter_measurement,__version__,meta/instanceID,_id,_uuid,_submission_time,_tags,_notes,_version,_duration,_submitted_by,_total_media,_media_count,_media_all_received,_xform_id -sdsa,vFm6aq2HXxNpNrqjN9Du4L,uuid:3323c78f-f070-49c3-b955-5a6fa934b17e,53734025,3323c78f-f070-49c3-b955-5a6fa934b17e,2019-11-18T09:28:39,,,vFm6aq2HXxNpNrqjN9Du4L,,davisraym,0,0,True,464257 diff --git a/onadata/libs/tests/utils/fixtures/bad_decimal.xlsx b/onadata/libs/tests/utils/fixtures/bad_decimal.xlsx deleted file mode 100644 index 5a596d08714a60438118791fc393a72944f81b8b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5846 zcmeHLcR1Wzw>CuYT@b?het&Dd?^;_+73&fe1|A+B1`DyPCdRp-Lx1*$ z3s}2adf6fFy8ZRxD!;cQJf+_h(F`HN{X)4)qCaJ4L`Nx4W!A3&A(>%AFvL84Pm(p^ zKcFIN=LcYN?2rt#<4G1GM}8Fqq`tXC!gaOJ)<>;zqi>{*)%m3yER?_udCBDQCC%lh z)w7?#ZgO#1fNV9H5V9*6AhUeMB`Fq) zJ;?s7%+*mB6Cs@fCXiBK+;HNx;`#XYvCC1~yw&r#3(#I1q-bMw#WT8$ju^6*J-Vx4 z4UfwHcPhrKcCtfl-iF=R0F|6sfv8vXrs^*b74I6Dv=q}${3$iwO*|2?YAo=bsnb}l zFDV^Uc>2j~dVC zXsesv_Xd1v50&@2;_2N@Xy=bRR!Jrvv2tROUYaK|V&h%59{uIX3+Ft1$4Fr9aXW90 z)7=D=JV7>UCX%(*h_qxiXR?pNg9btd^Kt(iG7sD))y=&Md z9Qf?i?>nqaLv}-F)zyv+6CZ^wOQKSr^-?k<(S;S4K*xsyb{e|eKfy#y8S8x5p36iY zf06)Yej^KiUDP-F6#k9$v^f2|WFHV=VqoN9|E@S$f9Tw~j52qH!O!Ov|LOP0K2?Hd z2pP@{rS~I^Pg;EYFoC&-+FrE~dIn%t*-FL6WT!d7poFPQ~8c)VN zHl*zRawiEz@1#pT77e9QWO+H*G6PS&zCc6R>b;mt@;&RzDDCEF$Z10IAtfAEuT;2( zFD+4yWT>pS9xbC1JRHi?oD0}V>@?zIsi?g)%$ItMGM&~8C($q^9^IY`=?84o++n{; z6bE)b9;pyIj^S!Y!>1@Hk9 z*$Kvi((QHi2Tk>RU~BZ0Mf(roZk^+Ofm%HHrs<=jk52BL^r05EGZI0z&rv3YL~WMA zm^pd;)e_cAd!>m(B-q2``eYf@UHxH(+tC54*Bzs(pxxLQTB7ptZXOR&+0?PEp|v&y z4RK>ipDY|mJ`RV*K;cNt(yXQH3Bf$>>|h&w`9!wdRl`9A!G&J3mP;;V(*|`_X~Ls@ z&Pd&?Vt!Ml%4DVJ2hd?%QJgVTY^1v~NVyx8fBQiZOB*ijL`u5})1Xb>Rqp3>JV2`@ z@ z9lzvb5|QO1;{N%<(vwg7UMs59v*$-0Wp~)Eq>avSXpgf`$t0r zE=g(yN~0@n6;7U&Zoy_3JnpcWNL%u7jD2F_;M<$^I&X&Kk0kps`)Z7xx>;`;iIOF= zTRxFL_MDds3b9Y>;6kuaq)=(Pwe1?A(s(@00-FZLOX}Br5e$?2{wy_NRVQ*~4KD4p z(&ls@nuI8y1-H!iIFLVW*%MGbKC6a)r#zlkbQYvs`dudcwi?dz;qR)UijdrHED~>0 zMXE7c4fv6rCG6?c6r75X-=z0@x7c^sQ8I@o2T;KQJTHl*pkOHV!mK%4%5kqz?q-%t zStyv7D!oK>;W>N6GCNTgAukKN5*^eXm*pY7ZF^%eYb>(&+hr^I`vQywJiFj<-f{dB zN0~}N!z{Lag+Cj|+6JVn)@ro}1`5X2*pIzgDmoOo6O^{4Zs8}t^-6m+6NarF57kHc z0#3^z1h7M0@gK{9>W3t_*;=~7taaU8Py3%!uf88MuF@hy7Mj-Rb!T@9QDjsI=K6qT z76-~(GBqr>DlfaP?8%P&wvEW=1b&fJcn zWjz9Dq{15lmRLXLYh9W;Aew>jxK3nEgLZg0MO}`zK303TF?HP{U1&69KL`4T5m|ng)n#bJay7T;Nf2+fI$owa5vT2lf>Q4>64iXAoQ&8o zt5vVZ?>Sub9WX08u~w8s9k zwh;e|Om`nR?2N@qy)MK!gzWpnW2`1WwNcK8?9o*7nV`gNGTNp1OwK2G=FB5uf0Sh{S>%Y=ir3uC$7 zh+I8;hmBZbh2k^dGCT;!aQLv(YLCx8dSJ2ct+j&(-HgM`;Ap}@PZy3c=ZEGx_s-JMAA@O>BDiaFqU>(|7(hlRLkPU^-0!5d>S90B;{AR_US1~$J$!z`x;i6Ty{@Yg37LWW{^QW=aiEfrH-gp&U)XX ze2a)vK{5QTppgD}{HN9eb#ihzYk6bUPZJ z_DoIBmKo;fyo%UC)X2Y?WJa)lU`Q4?Wq9?PWBqOyj1db@rp#6#X|62#b87>Y=Ve@% zz?bC3N{Q@z!dUrMtMWO5i#ppB^xtm;m68wU6rtF!39@HJn#J}XC>Iu5y3q>oZ{0WO zHugV!f1lR;2*{PXDefO+^mxn21flva=A}SOgqSNlX6Pg0Ew4Er*7TK*hU-!yHigM! zdW|4fVLnqU>6JKC+M3ThRL^(}*)co+7T-)xlZbk*()1~_k|y`Svz$f1!g&TI8AKdcYn z{~$cCG(|ccjSnob+@U=c0#Z|2tsZnHi2pVd=znCwkM8PxKAbs9J#tc|6+)(f)>gcl z*LI&tK}qY8cz%UD0H~n4YLc3=%}_%HFW-mjE)FeBos{JV0l+IZn<|w9+pmHy+j*WgJ3^Jh4aAg*H**ukcx&UMGxt!7Z1eB??(Eh92sU1f(8Go@F_v&iJy6*jM}=|mDH#vW|>=M*LS zKuNbPnR(LRxmws#FE`NxMt19@FbK7c_&CZqE!r?VD!By2JD(xsq^lb#pg|9AD0t0F z<}uy9Ea+-Gn8PaF^-RI3I@Uf8=YYR&Z{jV3*8OKi1DN-+2rbrDz5tVI5nA3LkF*3=dyLUWGX`4={6jHf#?G>zARya}ne|zZFqhTU8x5u0@S=>H)FfbibIswX zg`DI%k#%yQNEwP;Ep*|V<@XLiDq%wlk^6!iXkg}B$68l6J~O@5c(X>bnx%8q;cGo& zNI!_h`>R0VjMIqMTQJWbiV)mANS|$A#wOFy{F6Dv?Oo$GorgKtI6#bRlGk61Mj2KO z2839JT(5dpd@Woda4Kw=Nuw^XCSOODkagQ3%i~KtJf3agy4*AGOqGQyDUT_%9%@R9MiEu!hV$I5r_(OBLI;SNK%o9p>F!`w_yg}4pv z#ORp@?Zoo)4!4LH3kpEaOLHF$xQ`yNO_344oi}AVIFtZzQ)}nP$Ve5B%%waxHBRmJ z9JXeNcv;WW$_Wf7GF^`M?2^0JYmT!AV#UdA^nMg#8{G=`6?!fP;8JaiBG49kgv*tW zPLYKh>8eT#S=btOChsv82og9uT4qyt>#f`+XAG9LHqH>FqVVM90*~}@-N0a-!Y>Kzl$Fh96b%LLVeb!%fOq}jPbd3-@X)vpg(b^SZ?bftPrwqrc*bX z4~MkPJ=2D^2~Qt_{{)TeP4oj;J<~q`CKbldE4A}CM5kA3zs5gZuxY9OPf=9S|NA#6 zXRZ20l=B1euX{;Aw{U2biv!@s`c#R3#tZbG&RNjK wh38f2*M%WuXB+rS&@V1LUrDE;^)tH3|5n1ZRIzbRe?*LaIiX!s`)t*J0p749ivR!s diff --git a/onadata/libs/tests/utils/fixtures/good.xls b/onadata/libs/tests/utils/fixtures/good.xls index d59710c2851df8c9e1c86e67dc28f23b25045854..c6bf6c1ac539e91b73bc3284a6bebf4a35d19d8e 100644 GIT binary patch literal 16644 zcmeHu2UL{FwyuKY7LZ14kRYi=Bs4Z@K$0}Mi4p`P3Qf)+Q4pJ)gM@CBC`kb&gOYQw z36dlyQAv_>=+`*soN>J8%vuln{^yTYncQ^LPMeU6xz_?!x1 zpz66F5(xL~VasdgY~pTZZ{hOK2Ru#&GilqPTpKN&Gb zw#(j+X|+EpqZKIZt;uCs^)}2cx%4QaSx-@0Ol0O3;Ojuy#ZT9A;zaUEdK4aN_q{>{JA z5}(1pyQ4~Ye}1xF(fc z<6>!U?&`w({m-vZPTIuN%b*uFRNqWX}rO7fS#gcT6(hZ8|w zKD%4X5(w{8QTX2R=ztEdQT_2O2ARv7X4LCiO{B_Xczn7&XC1s`7Mv=fwfprTUI)9s z@I}HvU@OY#&Ew%Pw?bXR!)aMagLpGOM?fBVMD^`PKU4gCo^%X_J2Yi|rI9D_3;ZM(S(c?rkTHI2ej!m*z&N{d`g1$zO~((rI+^xqoP^2OINr0ivu4!n6LOovzO<;4@5JiZ3s^t;f~nwe2JBR;FkySE zmvJz-vUAyP^W;@A(N^>8oTAG7z43h{g~NxDlx^&cRof_&g9?^n>bZm-TPNX|*j0T6Yq>)oo>Hr2Npctgg;y zdqlNY+;@L@T=})!Z7(6yDLOtCx!Y}*wp?2*_PLf_TfPW4smS^GvmU#)Bnh+N9s^l7 zU0c$eV9%rWE$I7mS?k&^JnyQTHX9qRp1f$2VqfLVN%CT3b2y#(`9njsE-=^4ds6#}^`nBD;iMip=}t zx6+u*s(0PQiCt=qz62+wMG7&+Y<OgpCi@w(_AB_mHFLN?2d8AZla$H z#LTy%Dc@zFc3sb$XyfbX4jGct%f1f@|*1s>EJ*+KZkjeLW#=?qkwv zzp;CiVQUihGnAGjX_gL)TW-$#+Y8a_-gZRZ?-)9bipF2v-)$JFt2CNqyv0iqQT%!< z%^@0O;edVeX_zQp^kaWv?wO0c4woBaR@09@8+T}TS1$K(QcR`Pb|IO)6NuiG=v1q` zdiJK^iyRS~W_5bkkPGIXR{8a<{ZEr^oFfccj5UsVo1bprNm$nz>Losm&z|M9?LF*B zul|_2$4BnZwGnzvX}lyr=5Wr~VUKI|8`o;j_Z+FqSTklPhjm`p+vReSw|%{kg7Y}= zk&S*Lxh~_%YU)B`j953)U{pM;u9REp{mjhch?6h1%dF15&gShaC&6FZulI)Pjxpfu zICgya*7c!`>jbRSuJv&$RxV1{{`JW}zdl_OKvO6DaeNj3K&f6-sUC{UyWVyH_c_Bc!h>`LJ}{QYpQS z9NBAaoU(UlPI7Wjbh1dq<(0t=!RsSLuODJ7T9{)N8oMX`Lc~BNSjwyViEk?XL_dz> z0ATh`#5v)6+5W1Vt7H>JvD4FqQS*2?-6>y{eW}^wtt!@Tyua@5am~c}0w%R;&+K`Bd?U?GzcLc)tcDK%dIfLL-@bgy9uNC-gpqrCu zK53>~MLO;*!TH|#h}!2L2$WQC0ZVS{Z_R5E%YoOm!6T{Uq z%|hQ%e$7IEsfUYV_hIj*aR-hCMYcDw26no0l_QPkM~{99J!8P;q=V_Kj(BAnFe-1W zcllQj=(?Nh>LoskH=qCR0Xp>Z>b~bczhB8UTe=#KtexyV*`3tgeXfhkrx@`>4+Xw) z_~?Ics7JLv*)*;$sv3r{SbxR(ea#nrxcCeOoEgJNV=+ZyCHN9106VK;As@KKC)D<`nev*{i2VdU>d6)TFe$=yWVa%+qjXackhg*YvTh z%a3hUqf4^vRb_oRCa%iuSVk;QFiu_YSdJYi{CrdhwiOuFUOv$#tI)H(>0O`DsNGhv zxu^e&?Skzqk9w^J?Z%3(J$-iD1lw0$^;(VEtrbgq`U18?X0JTAHjXV*SM?`-Ke@i@ zuEe4re_jY%4cSJ>3fqH4?WbN!n#&63fJJkr#-wmhQ()1l94ly3vG8cIutCbDsDaL? z0ot($M}X=>K<^+TVBWiA-aAo{mHsH6;i$BnHY2?A?LCVR5oBoH0#vp@X*#MEt*Zz9 ze}3gFq(vek1Rm%?2{9m>wlV|&%{Lrjg#qzDD4-oc^I5jtgXRc4=!+;uz-`*nX&+#c zha;q*ZRnpA%MtWwr{M@2j4=Ov0qqFd$+ArsS|l*v7g371Z_}1RYk{d6j*y1-pq&b7 zgAvaJ=JlZD7-5^X-zlQ{&?J`Nd(aF4U?2Pq;*JeC9c7FO7=}wi8_^_%sK^ zhT+y2Hvaem)F4{H608f&7l`kJmmq$xc%u;Y1Q8~1R}V^np|;5>Ly)3x48yH3)ckh~ zP#@4YEOYKbvjy(P%{gYxjf(~sp47b7X^A8lDhS64*Il9n7 zfq_1FDMHUCCj*7RlnukBp%}DPAu0%g78uZjUc~U*B;p25 zV3@I*ECeP8p`-$Ek_5?<5b8aV;%lT7Xp~fen3iSaKrrIVV308G-QLDVb&br!%2Vx(RGvuS`oorzi$ z1QrdUWCL(g1<6wpq64JVn<%xKBoiTdN+1?EhyzHNO(I=Jg74InE@BdL+az5TM9l$E z=K!#gBIyE|SpxkPP1JHAuv`cwCxG*Eko;vry+EnrCd%?A$s$Oe4v6I*h=YcZolKga z48OlA-P$C?)+AjYM9l+G=K;JRLyiS9&jtE>nW)u4VD%762!In5Bo88_3zE9jOnJFk zvJH~=0K{Sj;-DqW21u6y@cEk4(Iz3!Owuht)cgQ-egNA=Bsz#WGsyp?iP|s(HUgm( z1aMvrlD|qg79{1=OzGAvIR(jc0I@iNIOqx4$))+p@nf6QyG%mfo20vds6_zkB7hg< z*D37c!(=iZf}V;3`b7a3D3D>n%yGf~pH0-ZA+Q}tj2K{vF^HInuq#+nThbAx zVF=SqRPrDk-VcZK1LM)5H_)M>2vZFxpA3`_S7r^)u>^zBp$eE5YD^0Z%9oEy#wp;+ z1b}z>fj~^li~lAgzIFut@b}1@yz5H5c_W6xRr^guMRnqbmyJggx1Ddu6+#G z;5%LViJm9Zy?1IeS{oU-#T}K%c2a4Ggv!%p@+-@Qt6me-7YBM-gd_F!*S+=ggerxr z-VxN-27390Bd^|H7rmcXStVT6PEg+&=#>P?JG8ZkPgK0@R@*x zLX%`6nqW{9OK=351c(X9_{$4i!31QX9Ie6k`GHZ0rhJrRKU^41l8b1PfzDcjpP@GkdQ8Aw=xip+ksqjn2?$3t;n*ht~@(lCru!NB@ad155Is8O-Hc6 zpqDJcPtc*17+5-r*cyC?AL##=7tr7bh9X$n@b$p> zH|%E?%W|V@Q}$+;nRt1oe0D!!;#DP1Plw{CQ)~B+YIT^zj!xr`UY{PHRMG8?ck5uQ z=yn#d_-7|1n=~RnP1JsKwEoC#wAXct@slxxDOUUV&j~Q*TdQ)=TG})ygc|unAR9h9VD}bEa zK~f%?SLo$!!%S~B<(}V3>eSUj9kUw;^_8#VBh(*>U&^5z*9EI z)3Ms-I13t>LFntS9pD)Tg?Y=$rMMPy;!cYG=ToAWa1v$?57e9;99?)HI5?Z1owT)! zomOeSMjf2i;BHyv;4k15Up5(t&_O2)XKh3^6tgBuoG)7)S9(`SH_5Pm)YCqzW=Aw> zZZv}|$&oF)#(~dTpqW$HzvY|cphm+Y6z4uH87WI_IeuTOiO)prw%2ll909Wu3ozMH zK&0g>s7;Dqa!ZY^>Bah_7v{m(A&n@HxCfbsOozZQW%Z}nd4=S6tsyFkH)yhtwkNq_ z@T7_8;DBTrYV4a!HRM;wK7YEBuSL&rAJ#j2heC?mczRF9sK;_CZEhs^DEA=0`qn_+ zh4dT|fu;Z=Mk$0sL)6Ig_@vEU`+8|iSf8?rmCCTUbF*U8*TcdlzoqN5PIix=7Y8Gy zh19!E0JPyL{z@oUOLIGO-tYPM)3z8L1BdZz)F%W-6d`*< zOvUlS)*1qfnPRr+GTSj|aWN*Wn83^cdjffAdpx9V!#Tw3-(8*5T;0GvBIE}pJvHW$ z?-}l^E@5uqde7PvCvbf50P95J9}^-I@)_`YmX&55u{BFqXch(qJQW4rEDOqi+S*kD z%X=?>e}MpD&DG->^kQkVB}~{$%>-&)T_IL^3kE7UIZ!&@5(YEZbvNG)LKQr5gvV+Q z$|#baKQE1%^N{CD#Ix^VlTg3!74m^n!%8Dw_b#T!`1vU3b)tTOvK^vAO!?5c zJo>ra!&goM*WS9Pg@4iB2&yu?eNoaRPn+-A_&D$LRf9+YNi7$Ux5ZJ#NbA)N3w)n9 zllCX+$h2|~hsQ=^E^EW9Gqp5p6bB9t?(aN_)++Z8)KX+1dv851D&8>J%L)@x6pRth zO-l=|#*{J7PwRwGH zKc@5y1y#P6u^}lb!kgaS7$j{8_<;0zu^-;o0#i{a(~+I*SBadFd5PX}Ww_rIYmcAW zZ3N2+roL=VC)XPjBI3#-8B3NRBuuNX4?2*%qxPxqQV#H=dq{>@Nxmwq(4G zQ`X-m5ywA%jjVeXz^9wkVjIw6x_gT%>d_6FM&j`7Eo!pvjxZ7aa|xGQ_BdrF148EV z-rrsD>-p9S6=O%ZkACae#9K2(eT#uq=qCjfU%X2_TX)b@P5xv<^IaA{O?PESxj@ZT zj^*73q=*aDD3CsTpeFRD=Iur*VoK#fY4)e(eOQ-L>iZp{Y#&H-f?w`fz)Z2!y zs&?9RgCHU#CK7#Vw(YaM){247jyk>P>S8;9TSHTadh6EfnYWXDe8U;2T^;jC1rPXc zQr_EM(H$O4C|*f&P3u%q$JGMFe8F-5JDz)Esyd+~S1usOoS%)^J{!|qt(rkLB-@K; zq==$0I@-w7qVfp$>~A}#4bQn#{LXqB>1Shs8@Pk*)#QH^wy1yYX)cx~&gN!!U7gP^ z3w*yLkThO2CrBNXmcB;mq)+P;75OpZWvatxQ~pA)E2G8u>Y?%Ao^LCjhja+ptM~j@ zo~UIO9CsJ&98_`(u89i-uZ-?VPX^@@=+4P}mL7d8vXV;U=W|SuFCz#8KKrGtQG(m> z_>D*9i9Hm+r<8Bvp||p2W%$*Q+P6m!g3QMcVx~0hrjwHyk0!0ZdO1wI*%KEGzTA4R zi0*E%jCRF?kVOl21G5KtSlW|lBUWJ}TOsM%$rd|Ez&HWXPqBqxC^GF<=O8V{%kVzA&ao}sdBi)7_*syzbRkA~ z<*34L>Y6~tL}hO|(Fe0{;xxrUj7q-fi1uMr(z5Y~^HiPQjlspmeTk9L2(94xF3yuR ziSt5(*W?GQw zW3fQfrc1-bThXwB;`{SuDAQvxgbjeSn(B*9+%(P6dS&#g6RW|1fhG2^TKsr-cH|nC zy1^*}(*sX(do&htv>!Lbd~~bJt*Kvgc|eA9mcE;0E-YJVLD7tZCNEPFk|>&cdFIs7 zbjj$_9nfsDiV8SH<1xXFI%;@K#FBYaKJ-a6ldukNU(!Nd)M|ahqb~z@MqsZ;uYQRs zx}h33KBv=Tc-MmdVTZRivr3Ijm`H?&u4$T1K%%xPLL$z?v{Q27Tz_qwMN3j7{)Wx` zN5yv9FY?-PxrcV4PX%abc_L@fT4V65E3qwi=jk=cZQZ(Iz|(iima;wy#34o`m!zsR z;;vydqVL}ejJQmtkaZ(B8&aJL-j<&WjX9TqUpm2Z!N4uJFJeEAk}~a*Y4!C4Oe{+f zb5DdE9rTf_EPL6*JD^{bzxuZTnJKdyv?__EjwYG;S?ky?w7d@B3|9`c3#9Kg*W-e- zu*wG#yM*N;%Hs#feb_c&97GBDI+H9HR^9()zN?tKwk+h6BHK+(@*9!AYX z8VF#NzkNv~-FfDven9%kH%jy+BPz7v_1y5%YEGAI%ANMsI|M4@Bw6wbivmfDpDG4M zquEl4{gL5#zyH8<^VH;$GQ8NhTv?awSf{c(+DRI>Z0v5>KQqVPbV@iicldQ`*E%-L zFolZ`SrmU1A1?kQKDc_?ntu-rgBF18}{d3r4NxV+@U;5sihBIu+vSN54sJCgwxLw+bs?pLFMbqkpXH$V^_;d^3*NzK1VMgR;-Q-rlA| zUaFEXxo0h`awetgkx&^!fGZiqVvHKXB(_Jr$GP?vddCVLVj%&n@5Ud=fi#!-#xFP5l4`muILtXdo6}1hQi|twtf7m5I6=_Cd48Uysmw`t!eP4 z5TL_r^W1r)bG<@1^{5+D=Avi1-QMSgLHDfKSaqd*cy0)HcNAu7Mf3Lh6D(TYgd#rJ zJ38bh&+e(V=DJKgB5WVI>308X{6JaK+gt~tlI5?B_Zi?rDG$>k{9vZXpR6v(jwX_| zYrSCr0E}rczS>XRunvjG-&YcN7BqwWc<0Vh5&aR}a9ljQwsy9XFm-US`92cGH%6sE zC?)IDv5$QUse(Bd?bSN06Uwv(A69COO))?pUtW=_Q21EC)^dJu^1d4HN5sk(q2@PU zmTs4yH;4p+%{N@zSRhnT^RTBUr#qg`y)0h>J7Ix@8oPK=Vg=KMJ(kf#Ee~K{B9(L( zA3b;cW=5h{bz|6=BQ9&GY2X9R#1IYav*NkeH&0Cjk2=cLf(%GP8JZhz>rhp2T)o(4 z#7x6E+c}l+@}Zb=rI2vp3z^{RG{r%NPyk!7_8rQ%e(Wz6K&;#=#rf9*3OeV~ITwtc zj6D7&yPz1n2#S}QcLrx1#m5&NV(e}+eR*{EJ-bm|iNky=Hg=9SdR{26x~j0%gPWNs zjxm_FPJV=CIb+iIUY#L9-Ey0fyO;I*c;_Xx^WhPFW!sT*uP$f_u8f%vPH>fkW`<6h zyU5Z{A#Zgg_^*VjhQiw(PnzT^ zp?;n{yx<9Q+^NQNUg=8>8HCt-JeE~H&K?MCb0 z^lQ3%;O1UA!6W#^!d+z`hPB!#jTty~#rfpQ**rvB_HtnjZnl5;hnMW=es%n>>&745 z|9$0H7^ke#4518EZD8P9yLx>=j(BL1klBrF;app)u)3+LnX3N;qqGbVJB5! z?a`)3>4__YcQEDE?YJwtJ+vN?IUCxWBPRn(sipE=SBEl)I*CGj^bPG&Gwwb`m`-lT zW>Ye;zYaYlV$LjT%uhC<5TDcpJxv3?&EQJM=pmCp$9BHkTV} z)a1!hHQ=l4$tg-rKE{?0E=pdH=|7ZJZl5!?$2NSGW^=8Vr|EwfW914hcL*uO*N}}B zDUVj+q_!}sImSB!6bq}ZBEbR5`Gxb|Z6ZrRcgBb7YRPxdYnhFBFyCPxaZnxw&HLX#y5oQQF zT6a6|=tUv?x0mlS6b8AL>8qUxY+^rtt&YjQW68Dw&OxiR4&BvMwfdCV81~E1!_zTK zPTKW~zHe)HZUK7M9Z(1;(g zbnHJ(rcfnzBXy*1AeM7YbsUVW$?d12BwTy zTwrA+<-7d6Xe9iY+cjXxa8y$-buy4ULFFFFy0bFjK}kOYy9(adP_m3$Aw~@^&%gPg z9A~Fa(Ws{Quq-`A^joyUKvIbUQ0hbUMXNEu(;7U_AXrc?G?%wtL%b|Q;nf*T)m#?4 z-*6x-;(q`0F2>oyTVLA2-qqaR)kxFR(cH!0EC!crj-TBaK6!E?oiLJram54;CZ%|H z+g|lj-YW_9fgO)`HQq2)@pg4c>m6!-Po#+OBL1hOSl3oldLG@lf$x4pG0 znv&~Xe6ASMNSjyFe7`VuFx_wHDvsnVol(wOw}QP3^ErwMyL=?CceJ$A-HUR1iBAnp z&a7K>wRf9k$eMq8m3(MTZ0^G6%hxGVuovBmt_5__G|VZHp%hcK@$78{Egs#OYC?%Y z7l-9(ime~ymH8L#5nRZYoz)Tn7B-f$9h;ew@z*7f#KozH5CK z5Lh!0y;Agh@?tzY#qOrqw+FTiEj!{SkpM!_X)X;XbEfA!NAsQFr2{HQFYfk7$;3*t zw_j8TT2}s&6j z?YHD~^WW^N{jC1uTK9~?zoj~^@Adh4C+TMcKTd|vmc+j$4#;-{|7C&vvxOh$H)oUP z-;zB4uNM9_h5lLf#}V>u0`yy|6#N&O|C|o}to-9>a5jkkmfi?|SN;z}>d$t545Mek z^0#ze#Q_W;QUr1aA%utm8Ug8wiYOon zD7{FNCcTJM=^)L}0wE+X=)61QynAQeH}Ag5T6>>=?fv~_fB(PF&e|vZHX}Zik@LLC zUM2ws5D3JO&+J3sJjJ+|{=^{|6@bNjk>129qQaw~LulFCJi9XVg%dMgMF8)(#uN$5=@*H!wuXb zt}N$O0xOg2K_A(*fc4IbW&0{BF>IS;uT7;k6&UA@0K^I0%tv7LqaZ9@`&P(KI*jH= z>y3e>Ymt{+pI+4+8nv_xq@`DVd<2w zv+MMj$<(=+Jd_=<_^BN~RPZ9kzA|8aJTGq)pp1VSjC&pPnYaVd)kXQ-I1%D~qFqeE zcx|DxrHVGuplb`;npiideC!ratE4U`Puwc{psQO-ib-hK4JdmrJ|g-aHW=(xLF1-w zEnw_1z~$GXYAGK|bOnuRt9NKsaecizZ@Emiq_hOd4%)yi%Jt`02UM2D?iVFAHi(Zv zx7NEMx)r7CGbB_A8*RQms*1KE4v?3g23eAp0Loa%qaDgLkzACptW9mm{s>FkAvSea zh0w_4pQ}hC{$^D2thjDDYI7WeOhb{#+w+4P+t-$_(@2846~fUZm^__wTD$$R5KUrJGB(8B(auz> z4jZCDlv}D-g6Pz;R;6r;K*oK>*;5)@0Icq~p`9s(6+-qIxA%5bf>hRFZ>qlpbc8fg zdTD_wKI*x&dFM&5%QR9`<5(SaklufwrsM9{zT6?v`;wh1h!h65oj{jA?XVedj4hx- z=vrl?DR)KMmVobl+OAwg2NrC9maYj=ui)0vNYqFfWdXR&T)7~d)u-=ldqs$+)VX0U zP(Di=Yo<)FK=cO2q;=I#cX3_asy_YDPra4fH2=*+pbl#{O0%7mmo7Z2t+(WF7Ast& zkwP*Xx!b>ZzxEnXJSIoi*tsxtU)+>DLw9+DTAwGS)^58|SufI7BaSqqRBvpjT4Ic- zzCn|~mz`R3%KT!LT|?E*%H#p=m@RwLlU&?emUK;H%DkEsb>pb-4Do!WhSj8(3U$JH z^85%uUyjl*NMt&1!Q1HoS#DD$Dc#JijpKK0NV2cT$(?h>)KS9vXo!GVupd!&Vw6Ul z42U+{E0~~HJs0wIYOyM2s)$+1I^SPW2tcvwqKX`N&WKbEzvFdd_4oCdgO?QtPm_}? zoID}MLB$WM@Pk)Nc6RU+omApRV`b%$KB5+FZlkW0JSBdr4W(9s+pE2{HiNrwUu<3kQ0XyHTqgh?9gQoLKPAUL5Ow2Q< zYbk4GGTIg$@=&NJ+k7goCEww1|agZ}s^EPxRRwm_HyIJ(cqvxZk z&qot?d)oDQ2x-IXF`!cdYk+!7bCagB>7@`|xnU5`Xq6_isN90P?-UQg_yow4`?#$f znIVGCVpXF-K}KQ)Z<2ta85rPt`qx}g;kXng<>E*U1V|0VN-IsO=bCm%5vMv z21tt)2=qb5*eiN7juGCUf;6*uH__#y|#UCXVE3*X3&oPMS zv>KNTp`0@vPIVbhbQw;aTI3Djqw;YJR9f#YmhK)?TyYz7U=7rs@8S4f?CNZt@3Ny;h4_9v*CqVo~N^{Cep}8%nk1i8d z>Qw3*Ld(RmnKx5uov23?N7o2WWVVm!?D2i)$TCn4DXx%&jsN#iI?xjn!;Q^U)yRa2nwOF-BQ{6 zGHqnB_U2~qT*nye!hH7=gN;!jMRY^CzaY_x@Iq5FWf}=H9fP%wTNZV@uQ%F>QEm~$ zvOUYTW)O?@C_+}dQeFn`dV3o~cu-(!YFv;qF=(!G^{4b4wfBdRjeW^X$YGz7UaDs{ zr;TS}+)iO!^`-ZPE2D)z{mq{)5eF|Z+Uy&Y2L|OGvYl;ukv981E_wjjba+H=lBa5w zrz*ZTHg}^8GHbTc>zzGVWPAPQ&etlVu$v+8qZhso-|-su@Ecyp%>{kzasAkH%+siN ze<6Q2#6h0qRFdtM@C~3Q$Y_}csX8)6@ z)>6E}HKQ=syQ)ssaOTzm65C@&pfgFx%@qNEstWJ;fOkFV?)o^`~TlvSh$FH+(b{f5sZ6mM68@o7y{?LturgmV*h1o+fu9rGAr+Sh3Isa#6dG2krLF z-ymzNmSj2vyZ1BC75c;x#xG1jA2>?i2rnyz$dE#SSix~jCeN7I6{|BIUBM<@k$@n2 z_@TT4P=S3>4{pmO-9~3!F~5j6yoB%A4`pMImt_Zc-8S;9=JT%Bx5LZA5E(E8hy%Qz z*+iO|{a$rO;}vZ46$v<^M+nM$5c-fg$|6k0It<-^#oQBbcpcv%35;3H4zz_2oN_oj>Y5|3;SYq#^x0)`HF-tqDLIcdk883isB2GIT(%xIhdpI zh7tG<2`C$Hyeu!+BHT!^hEKUh{~lh}7?EL&0P%tMvzkb=vY)HTkafVyJ4l!ydSsxy zhoKKyqq@Rn2Ex&Rx`TNt-Y^Z{0fDj!#LEhR`Orr9YWVKg=s(BHUO;4AK!EmxN0dc`jCBP1k%M_F-mned z0f(|3jF&wKeh^`Fu9nZdR=*c7>yF59M}UOEahxX4IN3dGGm0Isr4ABEM2{MjR~;(A z6~%{gk%C@RtCI}-$a0)y{Rt4)iKJ|@o8dz&smSovmQtJlasKDN!WX?ju+qr zBRGLR_VeLFx5LMhu;*Iqz^!#B;e?lo{TYe<^f6U_o1^@~t#z5-i~#=;D}QQ)I>JWx z3|TIJH+oAiWEqW9v)@ehw?99M8k^h!1l`QPXdX!^-?tawW?nycz;py1TE!Vg*5QC4 z>)wrG^+FUPa3sgi$oG#cg=?DzmA{gl&X2~9O^OO_+NfkLH+ijOa{P53fd3}qPnYoT zviZL^0{oMl{O_&&btAx^m&acVMd|u+s5}_`@026(hkE>L4*$B5!~c3~{5}-iXCW!v zVJ-k;s!IT2U9_}LbEnjvj5&UJwCMEdYK3X{Yx|EWs&ctn?5lxm8goxkJ zD=~~>foHC9kC-EO>T>oHTTTbSm*b@Tox*cY2tve4_&PY6Pl;5DW zF4EPp<1B|Nzf=-d53byE_Wu_E{0%<-%mwgwE%aXm0sO6O|IZ+Rzdz-A@LzY?Zn*(} zbG`kM0pR~w^n}-jM~SXqeLl4ej^qTN1V!5I%de9qVc9$7n-(% z4=CDkfO9CDymSaaEI2LnW%r(63UP{X8Qh9}{S)9h`sk6)MpXV_On71rl&PzIALWGzJ>fpVGzQmdND_x9yO+nQdC#NTh!s4y3xL)8c zl63*b>sg{8(HN=fF}LJf>Lo63dNNF}jDiSfCkKb`C^H&YatJbvcL#sW4fPmCfC*vVs0 zN5JuZ3~;n#4R0T4HO4T9U^{X2tTL6iK$RJ7i(OT1dQq{1;@Z=~DL*NTgy z0PYneqln>?_5cBJF@<58YVy%@B;)e&J&5Ri~_u3<2 z!Ytzn?+J0y@V5Io(q`*}d~)tk72AW=s@ZRZ)cUotGw(IBMEZYjHk6_!A3iEoDa}*| z3y(ZxD|b59^`)^=j{bW#?6b~?-4>5*WB5xU#IvVS#RWv(}WjD+2K~9#7(Bv7Z2{&|( z(sSp&0gJOkuqp-bIBFA;#zKEdfQIe14o~c5U|`{4_{Te9-?vXaxQj=jK+Dapfmi`1 zMuD{lV&i)%Ecje_$0TEYCD-RQ1r`yP$4jq$32PE+D_3L3+(BB)b->~A?s1>pw5lUt z>L=Ld@==@;?^QZFi(Zu3Z%ttG|1h4scEQ6A((lyz$l#a99=F|RlJ-dyW+guJ%Ms4} z{H*Jw1xtn%@8E%xCiAJ!4s`=flS!wV_j7@wjwN-DgN#>d+_Dwal-@{}z6z+0zCZ13 z>P9en&~OA&bH7I`H<|a{u^wUy`%HOYiElvjb{7L zQvJ<(H3tX0*5C99`B!kj&=Y9n_1CH0d`|@pl4abtozhKg>;$g+9vjm0R(hAjMQ~j$ zdUtybAFlzo(v1*5>77~r!m54$4(!#Yka(rBcsiM}p_leKl1d13hQkfQnue1HEf=}^ zbuX}@zRK=-8r+JWkWX2>Cw(P;@%q8qxuy98D)Vk3xwXl)AKAmepwIGKA$_Z(g1TcH z-_Mu|b<^(x*Kv+-0!~!(FyQ|=?(0JENrkAgan_*xJ>5p{$y#%uSKX>hgw0qZt36Hy zHxwDzvUybF^RYb02Xi`4kt|m)vgnUp_3JQuR|)sR6f|CD#E)n4myo$aU#!oiil1RU z@NjVGtWQ^P4j=Q&DlLWUa-ND;E~+?f7=5gW#s{}l{%Mk52*{Nh>U6rXWMCf}=ko2qkI-u&S;nfmZmB3G`fsp*@@N?63* zntSJ79F!C}6yPQmdCZW>wO_D_H~f(proxNX5`H;qpjVgkCerR_nH*5Y}0bNy|k%2*q{`~ECKpuyc zm%)|jaER{$uRUp(2cnKagPAQKRzX!T{^ix zx11g$nH~rC52{YMFv;SbG{5bCM388Ox%1;RInb5?2Iw8Xxsl5yOFF1X8OaK4? diff --git a/onadata/libs/tests/utils/test_csv_import.py b/onadata/libs/tests/utils/test_csv_import.py index 52d8bffa18..d0ace42bf4 100644 --- a/onadata/libs/tests/utils/test_csv_import.py +++ b/onadata/libs/tests/utils/test_csv_import.py @@ -413,33 +413,20 @@ def test_enforces_data_type(self): self._publish_xls_file(xls_file_path) self.xform = XForm.objects.last() - bad_integer_csv = open( - os.path.join(self.fixtures_dir, 'bad_integer.csv'), + bad_data = open( + os.path.join(self.fixtures_dir, 'bad_data.csv'), 'rb') result = csv_import.submit_csv(self.user.username, self.xform, - bad_integer_csv) + bad_data) + + expected_error = ( + "Invalid CSV data imported in row(s): {1: ['Unknown integer " + "format(s): 20.85'], 2: ['Unknown date format(s): 2014-0903', " + "'Unknown datetime format(s): sdsa', 'Unknown integer format(s): " + "21.53'], 3: ['Unknown integer format(s): 22.32'], 5: ['Unknown " + "date format(s): 2014-0900'], 6: ['Unknown date format(s): " + "2014-0901'], 7: ['Unknown date format(s): 2014-0902'], 8: " + "['Unknown date format(s): 2014-0903']}") self.assertEqual( result.get('error'), - 'Unknown integer format(s): 20.85') - - # Test datetime constraint is enforced - bad_date_csv = open( - os.path.join(self.fixtures_dir, 'bad_datetime.csv'), 'rb') - result = csv_import.submit_csv( - self.user.username, self.xform, bad_date_csv) - self.assertEqual( - result.get('error'), - 'Unknown datetime format(s): 2931093293232') - - # Test decimal constraint is enforced - xls_file_path = os.path.join(self.fixtures_dir, 'bad_decimal.xlsx') - self._publish_xls_file(xls_file_path) - self.xform = XForm.objects.last() - bad_decimal_csv = open( - os.path.join(self.fixtures_dir, 'bad_decimal.csv'), - 'rb') - result = csv_import.submit_csv(self.user.username, self.xform, - bad_decimal_csv) - self.assertEqual( - result.get('error'), - 'Unknown decimal format(s): sdsa') + expected_error) diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 9533225771..0a7a48148a 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -169,21 +169,231 @@ def submit_csv(username, xform, csv_file, overwrite=False): :py:func:`onadata.libs.utils.logger_tools.safe_create_instance` :param str username: the subission user - :param onadata.apps.logger.models.XForm xfrom: The submission's XForm. + :param onadata.apps.logger.models.XForm xform: The submission's XForm. :param (str or file): A CSV formatted file with submission rows. :return: If sucessful, a dict with import summary else dict with error str. :rtype: Dict """ - # TODO: Split this section ↓ into separate validate_csv function + # Validate csv before creation of Instances + try: + validated_data = validate_csv(csv_file, xform) + except UnicodeDecodeError: + return async_status( + FAILED, 'CSV file must be utf-8 encoded') + except Exception as e: + return async_status(FAILED, text(e)) + + if not validated_data.get('valid'): + return async_status( + FAILED, + u'Invalid CSV data imported in row(s): {}'.format( + validated_data.get('errors') + ) + ) + + if overwrite: + xform.instances.filter(deleted_at__isnull=True)\ + .update(deleted_at=timezone.now(), + deleted_by=User.objects.get(username=username)) + + validated_rows = validated_data.get('data') + row_count = validated_data.get('row_count') + additional_col = validated_data.get('additional_col') + rollback_uuids = [] + ona_uuid = {'formhub': {'uuid': xform.uuid}} + submission_time = datetime.utcnow().isoformat() + additions = duplicates = inserts = 0 + + try: + for row in validated_rows: + row_uuid = row.get('meta/instanceID') or 'uuid:{}'.format( + row.get('_uuid')) if row.get('_uuid') else None + submitted_by = row.get('_submitted_by') + submission_date = row.get('_submission_time', submission_time) + + for key in list(row): + # remove metadata (keys starting with '_') + if key.startswith('_'): + del row[key] + + # Inject our forms uuid into the submission + row.update(ona_uuid) + + old_meta = row.get('meta', {}) + new_meta, update = get_submission_meta_dict(xform, row_uuid) + inserts += update + old_meta.update(new_meta) + row.update({'meta': old_meta}) + + row_uuid = row.get('meta').get('instanceID') + rollback_uuids.append(row_uuid.replace('uuid:', '')) + + xml_file = BytesIO( + dict2xmlsubmission(row, xform, row_uuid, submission_date)) + + try: + error, instance = safe_create_instance( + username, xml_file, [], xform.uuid, None) + except ValueError as e: + error = e + + if error: + if not (isinstance(error, OpenRosaResponse) + and error.status_code == 202): + Instance.objects.filter( + uuid__in=rollback_uuids, xform=xform).delete() + return async_status(FAILED, text(error)) + else: + duplicates += 1 + else: + additions += 1 + + if additions % PROGRESS_BATCH_UPDATE == 0: + try: + current_task.update_state( + state='PROGRESS', + meta={ + 'progress': additions, + 'total': row_count, + 'info': additional_col + }) + except Exception: + logging.exception( + _(u'Could not update state of ' + 'import CSV batch process.')) + finally: + xform.submission_count(True) + + users = User.objects.filter( + username=submitted_by) if submitted_by else [] + if users: + instance.user = users[0] + instance.save() + except Exception as e: + failed_import(rollback_uuids, xform, e, text(e)) + finally: + xform.submission_count(True) + + return { + "additions": additions - inserts, + "duplicates": duplicates, + u"updates": inserts, + u"info": u"Additional column(s) excluded from the upload: '{0}'." + .format(', '.join(list(additional_col)))} + + +def get_async_csv_submission_status(job_uuid): + """ Gets CSV Submision progress or result + Can be used to pol long running submissions + :param str job_uuid: The submission job uuid returned by _submit_csv.delay + :return: Dict with import progress info (insertions & total) + :rtype: Dict + """ + if not job_uuid: + return async_status(FAILED, u'Empty job uuid') + + job = AsyncResult(job_uuid) + try: + # result = (job.result or job.state) + if job.state not in ['SUCCESS', 'FAILURE']: + response = async_status(celery_state_to_status(job.state)) + if isinstance(job.info, dict): + response.update(job.info) + + return response + + if job.state == 'FAILURE': + return async_status( + celery_state_to_status(job.state), text(job.result)) + + except BacklogLimitExceeded: + return async_status(celery_state_to_status('PENDING')) + + return job.get() + + +def submission_xls_to_csv(xls_file): + """Convert a submission xls file to submissions csv file + + :param xls_file: submissions xls file + :return: csv_file + """ + xls_file.seek(0) + xls_file_content = xls_file.read() + xl_workbook = xlrd.open_workbook(file_contents=xls_file_content) + first_sheet = xl_workbook.sheet_by_index(0) + + csv_file = BytesIO() + csv_writer = ucsv.writer(csv_file) + + date_columns = [] + boolean_columns = [] + + # write the header + csv_writer.writerow(first_sheet.row_values(0)) + + # check for any dates or boolean in the first row of data + for index in range(first_sheet.ncols): + row = 1 + + # In some cases where the field is not required the first row may have + # a null and thus XLS Dates (floats) or XLS Booleans + # will not be properly converted in the next steps + # As such we look for a row that contains a sample value of the column + # If present + while first_sheet.cell_type(row, index) == xlrd.XL_CELL_EMPTY \ + and row < first_sheet.nrows - 1: + row += 1 + + if first_sheet.cell_type(row, index) == xlrd.XL_CELL_DATE: + date_columns.append(index) + elif first_sheet.cell_type(row, index) == xlrd.XL_CELL_BOOLEAN: + boolean_columns.append(index) + + for row in range(1, first_sheet.nrows): + row_values = first_sheet.row_values(row) + + # convert excel dates(floats) to datetime + for date_column in date_columns: + try: + row_values[date_column] = xlrd.xldate_as_datetime( + row_values[date_column], + xl_workbook.datemode).isoformat() + except (ValueError, TypeError): + row_values[date_column] = first_sheet.cell_value( + row, date_column) + + # convert excel boolean to true/false + for boolean_column in boolean_columns: + row_values[boolean_column] = bool( + row_values[boolean_column] == EXCEL_TRUE) + + csv_writer.writerow(row_values) + + return csv_file + + +def validate_csv(csv_file, xform): + """Validate a CSV data being imported to an existing form + + Takes a csv formatted file or sting containing rows of submission/instance + and validates the data making sure that the data is valid enough to be + submitted depending on the unique contraints of the existing form. + + :param onadata.apps.logger.models.XForm xform: The subbmission's XForm. + :param (str or file): A CSV formatted file with submission rows. + :return: Returns a dict with the validation status + and data is returned. + :rtype: Dict + """ # Validate csv_file is utf-8 encoded or unicode if isinstance(csv_file, str): csv_file = BytesIO(csv_file) elif csv_file is None or not hasattr(csv_file, 'read'): - return async_status( - FAILED, - (u'Invalid param type for csv_file`.' - 'Expected utf-8 encoded file or unicode' - ' string got {} instead'.format(type(csv_file).__name__))) + raise Exception( + u'Invalid param type for csv_file`.' + 'Expected utf-8 encoded file or unicode' + ' string got {} instead'.format(type(csv_file).__name__)) # Change stream position to start of file csv_file.seek(0) @@ -193,10 +403,8 @@ def submit_csv(username, xform, csv_file, overwrite=False): # Make sure CSV headers have no spaces if any(' ' in header for header in csv_headers): - return async_status( - FAILED, - u'CSV file fieldnames should not contain spaces' - ) + raise Exception( + u'CSV file fieldnames should not contain spaces') # Get headers from stored data dictionary xform_headers = xform.get_headers() @@ -228,8 +436,8 @@ def submit_csv(username, xform, csv_file, overwrite=False): missing_col = sorted([m for m in missing_col if m.find('[') == -1]) if missing_col: - return async_status( - FAILED, u"Sorry uploaded file does not match the form. " + raise Exception( + u"Sorry uploaded file does not match the form. " u"The file is missing the column(s): " u"{0}.".format(', '.join(missing_col))) @@ -259,7 +467,7 @@ def get_columns_by_type(type_list): 'decimal': (get_columns_by_type(['decimal']), float) } - valid_data = True + valid = True row_count = 0 validated_rows = [] errors = {} @@ -271,30 +479,29 @@ def get_columns_by_type(type_list): for index in additional_col: del row[index] - # Remove 'n/a' values and '' values for xls + # Remove 'n/a' values and '' values from csv row = {k: v for (k, v) in row.items() if v != 'n/a' and v != ''} # Validate that row data row, error = validate_row(row, columns) if error: - valid_data = False + valid = False errors[row_count] = error - if valid_data: + if valid: location_data = {} for key in list(row): # Collect row location data into separate location_data dict if key.endswith(('.latitude', '.longitude', '.altitude', - '.precision')): + '.precision')): location_key, location_prop = key.rsplit(u'.', 1) location_data.setdefault(location_key, {}).update({ location_prop: row.get(key, '0') }) - # TODO: Maybe do this in the if conditional above ? # collect all location K-V pairs into single geopoint field(s) # in location_data dict for location_key in list(location_data): @@ -311,187 +518,46 @@ def get_columns_by_type(type_list): validated_rows.append(row) - # TODO: Split this section ↑ into separate validate_csv function - - if not valid_data: - return async_status( - FAILED, - u'Invalid CSV data imported: {}'.format( - errors - ) - ) - - if overwrite: - xform.instances.filter(deleted_at__isnull=True)\ - .update(deleted_at=timezone.now(), - deleted_by=User.objects.get(username=username)) - - rollback_uuids = [] - ona_uuid = {'formhub': {'uuid': xform.uuid}} - submission_time = datetime.utcnow().isoformat() - additions = duplicates = inserts = 0 - - for row in validated_rows: - row_uuid = row.get('meta/instanceID') or 'uuid:{}'.format( - row.get('_uuid')) if row.get('_uuid') else None - submitted_by = row.get('_submitted_by') - submission_date = row.get('_submission_time', submission_time) - - for key in list(row): - # remove metadata (keys starting with '_') - if key.startswith('_'): - del row[key] - - # Inject our forms uuid into the submission - row.update(ona_uuid) - - old_meta = row.get('meta', {}) - new_meta, update = get_submission_meta_dict(xform, row_uuid) - inserts += update - old_meta.update(new_meta) - row.update({'meta': old_meta}) - - row_uuid = row.get('meta').get('instanceID') - rollback_uuids.append(row_uuid.replace('uuid:', '')) - - xml_file = BytesIO( - dict2xmlsubmission(row, xform, row_uuid, submission_date)) - - try: - error, instance = safe_create_instance( - username, xml_file, [], xform.uuid, None) - except ValueError as e: - error = e - - if error: - if not (isinstance(error, OpenRosaResponse) - and error.status_code == 202): - Instance.objects.filter( - uuid__in=rollback_uuids, xform=xform).delete() - return async_status(FAILED, text(error)) - else: - duplicates += 1 - else: - additions += 1 - - if additions % PROGRESS_BATCH_UPDATE == 0: - try: - current_task.update_state( - state='PROGRESS', - meta={ - 'progress': additions, - 'total': row_count, - 'info': additional_col - }) - except Exception: - logging.exception( - _(u'Could not update state of ' - 'import CSV batch process.')) - finally: - xform.submission_count(True) - - users = User.objects.filter( - username=submitted_by) if submitted_by else [] - if users: - instance.user = users[0] - instance.save() - return { - "additions": additions - inserts, - "duplicates": duplicates, - u"updates": inserts, - u"info": u"Additional column(s) excluded from the upload: '{0}'." - .format(', '.join(list(additional_col)))} - - -def get_async_csv_submission_status(job_uuid): - """ Gets CSV Submision progress or result - Can be used to pol long running submissions - :param str job_uuid: The submission job uuid returned by _submit_csv.delay - :return: Dict with import progress info (insertions & total) - :rtype: Dict - """ - if not job_uuid: - return async_status(FAILED, u'Empty job uuid') - - job = AsyncResult(job_uuid) - try: - # result = (job.result or job.state) - if job.state not in ['SUCCESS', 'FAILURE']: - response = async_status(celery_state_to_status(job.state)) - if isinstance(job.info, dict): - response.update(job.info) - - return response - - if job.state == 'FAILURE': - return async_status( - celery_state_to_status(job.state), text(job.result)) - - except BacklogLimitExceeded: - return async_status(celery_state_to_status('PENDING')) - - return job.get() - - -def submission_xls_to_csv(xls_file): - """Convert a submission xls file to submissions csv file - - :param xls_file: submissions xls file - :return: csv_file - """ - xls_file.seek(0) - xls_file_content = xls_file.read() - xl_workbook = xlrd.open_workbook(file_contents=xls_file_content) - first_sheet = xl_workbook.sheet_by_index(0) - - csv_file = BytesIO() - csv_writer = ucsv.writer(csv_file) - - date_columns = [] - boolean_columns = [] - - # write the header - csv_writer.writerow(first_sheet.row_values(0)) - - # check for any dates or boolean in the first row of data - for index in range(first_sheet.ncols): - if first_sheet.cell_type(1, index) == xlrd.XL_CELL_DATE: - date_columns.append(index) - elif first_sheet.cell_type(1, index) == xlrd.XL_CELL_BOOLEAN: - boolean_columns.append(index) - - for row in range(1, first_sheet.nrows): - row_values = first_sheet.row_values(row) - - # convert excel dates(floats) to datetime - for date_column in date_columns: - try: - row_values[date_column] = xlrd.xldate_as_datetime( - row_values[date_column], - xl_workbook.datemode).isoformat() - except (ValueError, TypeError): - row_values[date_column] = first_sheet.cell_value( - row, date_column) - - # convert excel boolean to true/false - for boolean_column in boolean_columns: - row_values[boolean_column] = bool( - row_values[boolean_column] == EXCEL_TRUE) - - csv_writer.writerow(row_values) - - return csv_file + 'valid': valid, + 'data': validated_rows, + 'errors': errors, + 'row_count': row_count, + 'additional_col': additional_col + } def validate_row(row, columns): - """ """ - def validate_column_data(column, constraint_check): - """ """ + """Validate row of data making sure column constraints are enforced + + Takes a csv row containing data from a submission and validates the + data making sure data types are enforced. + + :param Dict row: The csv row + :param Dict columns: A dict containing the column headers to be validated + and the function that should check that the columns constraint is not + broken + :return: Returns a tuple containing the validated row and errors found + within the row if any + :rtype: Tuple + """ + def validate_column(columns, constraint_check): + """Validates columns within a row making sure data constraints are + not broken + + Takes a list of column headers to validate and a function of which + is used to validate the data is valid. + + :param List columns: A list of headers to be validated within the row. + :param func constraint_check: A function used to validate column data + :return: Returns a tuple containing the validity status of the rows + data and the validated_data if successful else it contains + the invalid_data + """ invalid_data = [] validated_data = {} - for key in column: + for key in columns: value = row.get(key, '') if value: @@ -511,7 +577,7 @@ def validate_column_data(column, constraint_check): errors = [] for datatype in columns: column, constraint_check = columns.get(datatype) - valid, data = validate_column_data(column, constraint_check) + valid, data = validate_column(column, constraint_check) if valid: if datatype == 'date':