Fix an issue where select_multiple data within groups would not be im…

…ported

onadata/libs/utils/csv_import.py

@@ -12,7 +12,7 @@
 from copy import deepcopy
 from datetime import datetime
 from io import BytesIO
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 import unicodecsv as ucsv
 import xlrd
@@ -246,19 +246,24 @@ def validate_csv_file(csv_file, xform):
     return {'valid': True, 'additional_col': additional_col}
 
 
-def flatten_split_select_multiples(row: Dict[str, Any], xform: XForm) -> dict:
+def flatten_split_select_multiples(
+        row: Dict[str, Any], select_multiples: List[str]) -> dict:
     """
     Flattens a select_multiple question that was previously split into
     different choice columns into one column
     """
     for key, value in row.items():
-        qstn = xform.get_element(key)
-        if qstn and qstn.type == MULTIPLE_SELECT_TYPE:
+        if key in select_multiples:
             picked_choices = [
                 k for k, v in value.items()
-                if v in ['1', 'True', 'TRUE'] or v == k]
+                if v in ['1', 'TRUE'] or v == k]
             new_value = ' '.join(picked_choices)
             row.update({key: new_value})
+        elif isinstance(value, dict):
+            # Handle cases where select_multiples are within a group
+            new_value = flatten_split_select_multiples(
+                value, select_multiples)
+            row.update({key: new_value})
     return row
 
 
@@ -293,6 +298,9 @@ def submit_csv(username, xform, csv_file, overwrite=False):
 
     csv_reader = ucsv.DictReader(csv_file, encoding='utf-8-sig')
     xform_json = json.loads(xform.json)
+    select_multiples = [
+        qstn.name for qstn in
+        xform.get_survey_elements_of_type(MULTIPLE_SELECT_TYPE)]
     ona_uuid = {'formhub': {'uuid': xform.uuid}}
     additions = duplicates = inserts = 0
     rollback_uuids = []
@@ -359,8 +367,10 @@ def submit_csv(username, xform, csv_file, overwrite=False):
                             lambda: '', location_data.get(location_key))
                     })
 
+                nested_dict = csv_dict_to_nested_dict(
+                    row, select_multiples=select_multiples)
                 row = flatten_split_select_multiples(
-                    csv_dict_to_nested_dict(row), xform=xform)
+                    nested_dict, select_multiples=select_multiples)
                 location_data = csv_dict_to_nested_dict(location_data)
                 # Merge location_data into the Row data
                 row = dict_merge(row, location_data)

onadata/libs/utils/dict_tools.py

@@ -58,24 +58,28 @@ def merge_list_of_dicts(list_of_dicts, override_keys: list = None):
     for row in list_of_dicts:
         for k, v in row.items():
             if isinstance(v, list):
-                z = merge_list_of_dicts(result[k] + v if k in result else v)
+                z = merge_list_of_dicts(result[k] + v if k in result else v,
+                                        override_keys=override_keys)
                 result[k] = z if isinstance(z, list) else [z]
             else:
                 if k in result:
                     if isinstance(v, dict):
                         try:
-                            result[k] = merge_list_of_dicts([result[k], v])
+                            result[k] = merge_list_of_dicts(
+                                [result[k], v], override_keys=override_keys)
                         except AttributeError as e:
                             # If the key is within the override_keys
                             # (Is a select_multiple question) We make
                             # the assumption that the dict values are
                             # more accurate as they usually mean that
                             # the select_multiple has been split into
                             # separate columns for each choice
-                            if isinstance(result[k], str) and \
-                                 k in override_keys:
+                            if override_keys and isinstance(result[k], str)\
+                                    and k in override_keys:
                                 result[k] = {}
-                                result[k] = merge_list_of_dicts([result[k], v])
+                                result[k] = merge_list_of_dicts(
+                                    [result[k], v],
+                                    override_keys=override_keys)
                             else:
                                 raise e
                     else:
@@ -113,12 +117,11 @@ def remove_indices_from_dict(obj):
     return result
 
 
-def csv_dict_to_nested_dict(csv_dict):
+def csv_dict_to_nested_dict(csv_dict, select_multiples=None):
     """
     Converts a CSV dict to nested dicts.
     """
     results = []
-    select_question_keys = []
 
     for key in list(csv_dict):
         result = {}
@@ -128,14 +131,11 @@ def csv_dict_to_nested_dict(csv_dict):
         if len(split_keys) == 1:
             result[key] = value
         else:
-            if split_keys[0] not in select_question_keys and \
-                    split_keys[0] != 'meta':
-                select_question_keys.append(split_keys[0])
             result = list_to_dict(split_keys, value)
 
         results.append(result)
 
-    merged_dict = merge_list_of_dicts(results, select_question_keys)
+    merged_dict = merge_list_of_dicts(results, select_multiples)
 
     return remove_indices_from_dict(merged_dict)