Refactor external choices sheet fn to use openpyxl

Signed-off-by: Kipchirchir Sigei <[email protected]>

onadata/apps/main/tests/test_base.py

@@ -111,6 +111,13 @@ def _publish_xlsx_file(self):
         # make sure publishing the survey worked
         self.assertEqual(XForm.objects.count(), pre_count + 1)
 
+    def _publish_xlsx_file_with_external_choices(self):
+        path = os.path.join(self.this_directory, 'fixtures', 'external_choice_form_v1.xlsx')
+        pre_count = XForm.objects.count()
+        TestBase._publish_xls_file(self, path)
+        # make sure publishing the survey worked
+        self.assertEqual(XForm.objects.count(), pre_count + 1)
+
     def _publish_xls_file_and_set_xform(self, path):
         count = XForm.objects.count()
         self._publish_xls_file(path)

onadata/apps/viewer/models/data_dictionary.py

@@ -7,6 +7,7 @@
 
 import unicodecsv as csv
 import xlrd
+import openpyxl
 from builtins import str as text
 from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.db.models.signals import post_save, pre_save
@@ -69,20 +70,20 @@ def sheet_to_csv(xls_content, sheet_name):
 
     :returns: a (StrionIO) csv file object
     """
-    workbook = xlrd.open_workbook(file_contents=xls_content)
+    workbook = openpyxl.load_workbook(xls_content.path)
 
-    sheet = workbook.sheet_by_name(sheet_name)
+    sheet = workbook.get_sheet_by_name(sheet_name)
 
-    if not sheet or sheet.nrows < 2:
+    if not sheet or sheet.max_column < 2:
         raise Exception(_(u"Sheet <'%(sheet_name)s'> has no data." %
                           {'sheet_name': sheet_name}))
 
     csv_file = BytesIO()
 
     writer = csv.writer(csv_file, encoding='utf-8', quoting=csv.QUOTE_ALL)
-    mask = [v and len(v.strip()) > 0 for v in sheet.row_values(0)]
+    mask = [v and len(v.strip()) > 0 for v in list(sheet.values)[0]]
 
-    header = [v for v, m in zip(sheet.row_values(0), mask) if m]
+    header = [v for v, m in zip(list(sheet.values)[0], mask) if m]
     writer.writerow(header)
 
     name_column = None
@@ -94,33 +95,32 @@ def sheet_to_csv(xls_content, sheet_name):
     integer_fields = False
     date_fields = False
     if name_column:
-        name_column_values = sheet.col_values(name_column)
-        for index in range(len(name_column_values)):
-            if sheet.cell_type(index, name_column) == xlrd.XL_CELL_NUMBER:
+        for index in range(1, sheet.max_column):
+            if sheet.cell(index, name_column).data_type == 'n':
                 integer_fields = True
-            elif sheet.cell_type(index, name_column) == xlrd.XL_CELL_DATE:
+            elif sheet.cell(index, name_column).is_date:
                 date_fields = True
 
-    for row in range(1, sheet.nrows):
+    for row in range(1, sheet.max_row):
         if integer_fields or date_fields:
             # convert integers to string/datetime if name has numbers/dates
             row_values = []
-            for index, val in enumerate(sheet.row_values(row)):
-                if sheet.cell_type(row, index) == xlrd.XL_CELL_NUMBER:
+            for index, val in enumerate(list(sheet.values)[row]):
+                if sheet.cell(row, index).data_type == 'n':
                     try:
                         val = str(
                             float(val) if (
                                     float(val) > int(val)) else int(val))
                     except ValueError:
                         pass
-                elif sheet.cell_type(row, index) == xlrd.XL_CELL_DATE:
+                elif sheet.cell(row, index).is_date:
                     val = xlrd.xldate_as_datetime(
                         val, workbook.datemode).isoformat()
                 row_values.append(val)
             writer.writerow([v for v, m in zip(row_values, mask) if m])
         else:
             writer.writerow(
-                [v for v, m in zip(sheet.row_values(row), mask) if m])
+                [v for v, m in zip(list(sheet.values)[row], mask) if m])
 
     return csv_file
 
@@ -240,7 +240,7 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs):
     if hasattr(instance, 'has_external_choices') \
             and instance.has_external_choices:
         instance.xls.seek(0)
-        f = sheet_to_csv(instance.xls.read(), 'external_choices')
+        f = sheet_to_csv(instance.xls, 'external_choices')
         f.seek(0, os.SEEK_END)
         size = f.tell()
         f.seek(0)

onadata/libs/utils/export_tools.py

@@ -131,7 +131,6 @@ def generate_export(export_type, xform, export_id=None, options=None):
         Export.CSV_EXPORT: 'to_flat_csv_export',
         Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
         Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
-        Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets',
     }
 
     if xform is None:

requirements/base.in

@@ -2,7 +2,7 @@
 -e .
 
 # installed from Git
--e git+https://github.com/lindsay-stevens/pyxform.git@054435be7201e3c9459d83de39b4ae11e71dece1#egg=pyxform
+-e git+https://github.com/XLSForm/pyxform.git@f9abb96b049f2a52992c3e1d072a427e2c4a5b8b#egg=pyxform
 -e git+https://github.com/onaio/python-digest.git@3af1bd0ef6114e24bf23d0e8fd9d7ebf389845d1#egg=python-digest
 -e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest
 -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router

requirements/base.pip

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements/base.pip requirements/base.in
 #
--e git+https://github.com/lindsay-stevens/pyxform.git@054435be7201e3c9459d83de39b4ae11e71dece1#egg=pyxform
+-e git+https://github.com/XLSForm/pyxform.git@f9abb96b049f2a52992c3e1d072a427e2c4a5b8b#egg=pyxform
 -e git+https://github.com/onaio/django-digest.git@eb85c7ae19d70d4690eeb20983e94b9fde8ab8c2#egg=django-digest
     # via -r requirements/base.in
 -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router