diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index b46dcce80e..dd5bbdbfe2 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -5506,14 +5506,25 @@ def test_external_choice_integer_name_xlsform(self): self.assertIsNotNone(metadata) csv_reader = csv.reader(codecs.iterdecode(metadata.data_file, "utf-8")) - header = next(csv_reader) - name_index = header.index("name") - for row in csv_reader: - try: - int(row[name_index]) - self.assertTrue(isinstance(row[name_index], str)) - except ValueError: - self.assertTrue(isinstance(row[name_index], str)) + expected_data = [ + ["list_name", "name", "label", "state", "county"], + ["states", "1", "Texas", "", ""], + ["states", "2", "Washington", "", ""], + ["counties", "b1", "King", "2", ""], + ["counties", "b2", "Pierce", "2", ""], + ["counties", "b3", "King", "1", ""], + ["counties", "b4", "Cameron", "1", ""], + ["cities", "dumont", "Dumont", "1", "b3"], + ["cities", "finney", "Finney", "1", "b3"], + ["cities", "brownsville", "brownsville", "1", "b4"], + ["cities", "harlingen", "harlingen", "1", "b4"], + ["cities", "seattle", "Seattle", "2", "b3"], + ["cities", "redmond", "Redmond", "2", "b3"], + ["cities", "tacoma", "Tacoma", "2", "b2"], + ["cities", "puyallup", "Puyallup", "2", "b2"], + ] + for index, row in enumerate(csv_reader): + self.assertEqual(row, expected_data[index]) def test_csv_xls_import_errors(self): with HTTMock(enketo_mock): diff --git a/onadata/apps/main/tests/fixtures/external_choice_form_v3.xlsx b/onadata/apps/main/tests/fixtures/external_choice_form_v3.xlsx new file mode 100644 index 0000000000..7ca90f36d7 Binary files /dev/null and b/onadata/apps/main/tests/fixtures/external_choice_form_v3.xlsx differ diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index a638b61202..a0e6adeba1 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -132,9 +132,9 @@ def _publish_xlsx_file(self): # make sure publishing the survey worked self.assertEqual(XForm.objects.count(), pre_count + 1) - def _publish_xlsx_file_with_external_choices(self): + def _publish_xlsx_file_with_external_choices(self, form_version="v1"): path = os.path.join( - self.this_directory, "fixtures", "external_choice_form_v1.xlsx" + self.this_directory, "fixtures", f"external_choice_form_{form_version}.xlsx" ) pre_count = XForm.objects.count() TestBase._publish_xls_file(self, path) diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 2ab8df2f26..860e311d55 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -42,6 +42,7 @@ class TestProcess(TestBase, SerializeMixin): """ Test form publishing processes. """ + lockfile = __file__ loop_str = "loop_over_transport_types_frequency" @@ -108,6 +109,12 @@ def test_publish_xlsx_file_with_external_choices(self): """Test publishing an XLSX file with external choices""" self._publish_xlsx_file_with_external_choices() + def test_public_xlsx_file_with_external_choices_with_empty_row(self): + """ + Test that a form with empty spaces in list_name column is uploaded correctly + """ + self._publish_xlsx_file_with_external_choices(form_version="v3") + @patch("onadata.apps.main.forms.requests") def test_google_url_upload(self, mock_requests): """Test uploading an XLSForm from a Google Docs SpreadSheet URL.""" diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index 4de9a2eeaa..856ea96868 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -5,17 +5,19 @@ import os from io import BytesIO, StringIO -import unicodecsv as csv -import openpyxl from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models.signals import post_save, pre_save from django.utils import timezone from django.utils.translation import gettext as _ + +import openpyxl +import unicodecsv as csv from floip import FloipSurvey from kombu.exceptions import OperationalError from pyxform.builder import create_survey_element_from_dict from pyxform.utils import has_external_choices from pyxform.xls2json import parse_file_to_json +from pyxform.xls2json_backends import xlsx_value_to_str from onadata.apps.logger.models.xform import XForm, check_version_set, check_xform_uuid from onadata.apps.logger.xform_instance_parser import XLSFormError @@ -86,41 +88,19 @@ def sheet_to_csv(xls_content, sheet_name): writer = csv.writer(csv_file, encoding="utf-8", quoting=csv.QUOTE_ALL) mask = [v and len(v.strip()) > 0 for v in list(sheet.values)[0]] - header = [v for v, m in zip(list(sheet.values)[0], mask) if m] - writer.writerow(header) - - name_column = None - try: - name_column = header.index("name") - except ValueError: - pass - - integer_fields = False - date_fields = False - if name_column: - for index in range(1, sheet.max_column): - if sheet.cell(index, name_column).data_type == "n": - integer_fields = True - elif sheet.cell(index, name_column).is_date: - date_fields = True - - for row, value in enumerate(sheet.iter_rows()): - if integer_fields or date_fields: - # convert integers to string/datetime if name has numbers/dates - row_values = [] - for index, val in enumerate(value): - if sheet.cell(row, index).data_type == "n": - try: - val = str(float(val) if (float(val) > int(val)) else int(val)) - except ValueError: - pass - elif sheet.cell(row, index).is_date: - val = val.strftime("%Y-%m-%d").isoformat() + for row in sheet.iter_rows(values_only=True): + row_values = [] + try: + for val in row: + if val is not None: + val = xlsx_value_to_str(val) + val = val.strip() row_values.append(val) + except TypeError: + continue + + if not all(v is None for v in row_values): writer.writerow([v for v, m in zip(row_values, mask) if m]) - else: - single_row = [cell.value for cell in value] - writer.writerow([v for v, m in zip(single_row, mask) if m]) return csv_file