Ensure external select to csv conversion works as expected

onadata/apps/api/tests/viewsets/test_xform_viewset.py

@@ -5506,14 +5506,25 @@ def test_external_choice_integer_name_xlsform(self):
                 self.assertIsNotNone(metadata)
 
                 csv_reader = csv.reader(codecs.iterdecode(metadata.data_file, "utf-8"))
-                header = next(csv_reader)
-                name_index = header.index("name")
-                for row in csv_reader:
-                    try:
-                        int(row[name_index])
-                        self.assertTrue(isinstance(row[name_index], str))
-                    except ValueError:
-                        self.assertTrue(isinstance(row[name_index], str))
+                expected_data = [
+                    ["list_name", "name", "label", "state", "county"],
+                    ["states", "1", "Texas", "", ""],
+                    ["states", "2", "Washington", "", ""],
+                    ["counties", "b1", "King", "2", ""],
+                    ["counties", "b2", "Pierce", "2", ""],
+                    ["counties", "b3", "King", "1", ""],
+                    ["counties", "b4", "Cameron", "1", ""],
+                    ["cities", "dumont", "Dumont", "1", "b3"],
+                    ["cities", "finney", "Finney", "1", "b3"],
+                    ["cities", "brownsville", "brownsville", "1", "b4"],
+                    ["cities", "harlingen", "harlingen", "1", "b4"],
+                    ["cities", "seattle", "Seattle", "2", "b3"],
+                    ["cities", "redmond", "Redmond", "2", "b3"],
+                    ["cities", "tacoma", "Tacoma", "2", "b2"],
+                    ["cities", "puyallup", "Puyallup", "2", "b2"],
+                ]
+                for index, row in enumerate(csv_reader):
+                    self.assertEqual(row, expected_data[index])
 
     def test_csv_xls_import_errors(self):
         with HTTMock(enketo_mock):

onadata/apps/main/tests/test_base.py

@@ -132,9 +132,9 @@ def _publish_xlsx_file(self):
         # make sure publishing the survey worked
         self.assertEqual(XForm.objects.count(), pre_count + 1)
 
-    def _publish_xlsx_file_with_external_choices(self):
+    def _publish_xlsx_file_with_external_choices(self, form_version="v1"):
         path = os.path.join(
-            self.this_directory, "fixtures", "external_choice_form_v1.xlsx"
+            self.this_directory, "fixtures", f"external_choice_form_{form_version}.xlsx"
         )
         pre_count = XForm.objects.count()
         TestBase._publish_xls_file(self, path)

onadata/apps/main/tests/test_process.py

@@ -42,6 +42,7 @@ class TestProcess(TestBase, SerializeMixin):
     """
     Test form publishing processes.
     """
+
     lockfile = __file__
 
     loop_str = "loop_over_transport_types_frequency"
@@ -108,6 +109,12 @@ def test_publish_xlsx_file_with_external_choices(self):
         """Test publishing an XLSX file with external choices"""
         self._publish_xlsx_file_with_external_choices()
 
+    def test_public_xlsx_file_with_external_choices_with_empty_row(self):
+        """
+        Test that a form with empty spaces in list_name column is uploaded correctly
+        """
+        self._publish_xlsx_file_with_external_choices(form_version="v3")
+
     @patch("onadata.apps.main.forms.requests")
     def test_google_url_upload(self, mock_requests):
         """Test uploading an XLSForm from a Google Docs SpreadSheet URL."""

onadata/apps/viewer/models/data_dictionary.py

@@ -5,17 +5,19 @@
 import os
 from io import BytesIO, StringIO
 
-import unicodecsv as csv
-import openpyxl
 from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.db.models.signals import post_save, pre_save
 from django.utils import timezone
 from django.utils.translation import gettext as _
+
+import openpyxl
+import unicodecsv as csv
 from floip import FloipSurvey
 from kombu.exceptions import OperationalError
 from pyxform.builder import create_survey_element_from_dict
 from pyxform.utils import has_external_choices
 from pyxform.xls2json import parse_file_to_json
+from pyxform.xls2json_backends import xlsx_value_to_str
 
 from onadata.apps.logger.models.xform import XForm, check_version_set, check_xform_uuid
 from onadata.apps.logger.xform_instance_parser import XLSFormError
@@ -86,41 +88,19 @@ def sheet_to_csv(xls_content, sheet_name):
     writer = csv.writer(csv_file, encoding="utf-8", quoting=csv.QUOTE_ALL)
     mask = [v and len(v.strip()) > 0 for v in list(sheet.values)[0]]
 
-    header = [v for v, m in zip(list(sheet.values)[0], mask) if m]
-    writer.writerow(header)
-
-    name_column = None
-    try:
-        name_column = header.index("name")
-    except ValueError:
-        pass
-
-    integer_fields = False
-    date_fields = False
-    if name_column:
-        for index in range(1, sheet.max_column):
-            if sheet.cell(index, name_column).data_type == "n":
-                integer_fields = True
-            elif sheet.cell(index, name_column).is_date:
-                date_fields = True
-
-    for row, value in enumerate(sheet.iter_rows()):
-        if integer_fields or date_fields:
-            # convert integers to string/datetime if name has numbers/dates
-            row_values = []
-            for index, val in enumerate(value):
-                if sheet.cell(row, index).data_type == "n":
-                    try:
-                        val = str(float(val) if (float(val) > int(val)) else int(val))
-                    except ValueError:
-                        pass
-                elif sheet.cell(row, index).is_date:
-                    val = val.strftime("%Y-%m-%d").isoformat()
+    for row in sheet.iter_rows(values_only=True):
+        row_values = []
+        try:
+            for val in row:
+                if val is not None:
+                    val = xlsx_value_to_str(val)
+                    val = val.strip()
                 row_values.append(val)
+        except TypeError:
+            continue
+
+        if not all(v is None for v in row_values):
             writer.writerow([v for v, m in zip(row_values, mask) if m])
-        else:
-            single_row = [cell.value for cell in value]
-            writer.writerow([v for v, m in zip(single_row, mask) if m])
     return csv_file