diff --git a/onadata/apps/main/tests/fixtures/external_choice_form_v1.xlsx b/onadata/apps/main/tests/fixtures/external_choice_form_v1.xlsx new file mode 100644 index 0000000000..9626cdde00 Binary files /dev/null and b/onadata/apps/main/tests/fixtures/external_choice_form_v1.xlsx differ diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index a3d4781bca..abc963ef1b 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -111,6 +111,13 @@ def _publish_xlsx_file(self): # make sure publishing the survey worked self.assertEqual(XForm.objects.count(), pre_count + 1) + def _publish_xlsx_file_with_external_choices(self): + path = os.path.join(self.this_directory, 'fixtures', 'external_choice_form_v1.xlsx') + pre_count = XForm.objects.count() + TestBase._publish_xls_file(self, path) + # make sure publishing the survey worked + self.assertEqual(XForm.objects.count(), pre_count + 1) + def _publish_xls_file_and_set_xform(self, path): count = XForm.objects.count() self._publish_xls_file(path) diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index b721ee1bbe..39c0ea5d7d 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -7,6 +7,7 @@ import unicodecsv as csv import xlrd +import openpyxl from builtins import str as text from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models.signals import post_save, pre_save @@ -69,20 +70,20 @@ def sheet_to_csv(xls_content, sheet_name): :returns: a (StrionIO) csv file object """ - workbook = xlrd.open_workbook(file_contents=xls_content) + workbook = openpyxl.load_workbook(xls_content.path) - sheet = workbook.sheet_by_name(sheet_name) + sheet = workbook.get_sheet_by_name(sheet_name) - if not sheet or sheet.nrows < 2: + if not sheet or sheet.max_column < 2: raise Exception(_(u"Sheet <'%(sheet_name)s'> has no data." % {'sheet_name': sheet_name})) csv_file = BytesIO() writer = csv.writer(csv_file, encoding='utf-8', quoting=csv.QUOTE_ALL) - mask = [v and len(v.strip()) > 0 for v in sheet.row_values(0)] + mask = [v and len(v.strip()) > 0 for v in list(sheet.values)[0]] - header = [v for v, m in zip(sheet.row_values(0), mask) if m] + header = [v for v, m in zip(list(sheet.values)[0], mask) if m] writer.writerow(header) name_column = None @@ -94,33 +95,32 @@ def sheet_to_csv(xls_content, sheet_name): integer_fields = False date_fields = False if name_column: - name_column_values = sheet.col_values(name_column) - for index in range(len(name_column_values)): - if sheet.cell_type(index, name_column) == xlrd.XL_CELL_NUMBER: + for index in range(1, sheet.max_column): + if sheet.cell(index, name_column).data_type == 'n': integer_fields = True - elif sheet.cell_type(index, name_column) == xlrd.XL_CELL_DATE: + elif sheet.cell(index, name_column).is_date: date_fields = True - for row in range(1, sheet.nrows): + for row in range(1, sheet.max_row): if integer_fields or date_fields: # convert integers to string/datetime if name has numbers/dates row_values = [] - for index, val in enumerate(sheet.row_values(row)): - if sheet.cell_type(row, index) == xlrd.XL_CELL_NUMBER: + for index, val in enumerate(list(sheet.values)[row]): + if sheet.cell(row, index).data_type == 'n': try: val = str( float(val) if ( float(val) > int(val)) else int(val)) except ValueError: pass - elif sheet.cell_type(row, index) == xlrd.XL_CELL_DATE: + elif sheet.cell(row, index).is_date: val = xlrd.xldate_as_datetime( val, workbook.datemode).isoformat() row_values.append(val) writer.writerow([v for v, m in zip(row_values, mask) if m]) else: writer.writerow( - [v for v, m in zip(sheet.row_values(row), mask) if m]) + [v for v, m in zip(list(sheet.values)[row], mask) if m]) return csv_file @@ -240,7 +240,7 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): if hasattr(instance, 'has_external_choices') \ and instance.has_external_choices: instance.xls.seek(0) - f = sheet_to_csv(instance.xls.read(), 'external_choices') + f = sheet_to_csv(instance.xls, 'external_choices') f.seek(0, os.SEEK_END) size = f.tell() f.seek(0) diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index a05b32e60c..6e174188b0 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -131,7 +131,6 @@ def generate_export(export_type, xform, export_id=None, options=None): Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', - Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', } if xform is None: diff --git a/requirements/base.in b/requirements/base.in index f2d6a92b03..b2c8461ec1 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,7 +2,7 @@ -e . # installed from Git --e git+https://github.com/lindsay-stevens/pyxform.git@054435be7201e3c9459d83de39b4ae11e71dece1#egg=pyxform +-e git+https://github.com/XLSForm/pyxform.git@f9abb96b049f2a52992c3e1d072a427e2c4a5b8b#egg=pyxform -e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest -e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router diff --git a/requirements/base.pip b/requirements/base.pip index d36511815f..3200f0e3c3 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -4,7 +4,7 @@ # # pip-compile --output-file=requirements/base.pip requirements/base.in # --e git+https://github.com/lindsay-stevens/pyxform.git@054435be7201e3c9459d83de39b4ae11e71dece1#egg=pyxform +-e git+https://github.com/XLSForm/pyxform.git@f9abb96b049f2a52992c3e1d072a427e2c4a5b8b#egg=pyxform -e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest # via -r requirements/base.in -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router