Add Osm Data to Exports

- Add osm data to xls, zipped csv and zipped sav exports
- Add tests for the above

onadata/apps/viewer/tests/test_export_builder.py

@@ -201,6 +201,24 @@ class TestExportBuilder(TestBase):
             ]
         }
     ]
+    osm_data = [
+        {
+            'photo': '1424308569120.jpg',
+            'osm_road': 'OSMWay234134797.osm',
+            'osm_building': 'OSMWay34298972.osm',
+            'fav_color': 'red',
+            'osm_road:ctr:lat': '23.708174238006087',
+            'osm_road:ctr:lon': '90.40946505581161',
+            'osm_road:highway': 'tertiary',
+            'osm_road:lanes': 2,
+            'osm_road:name': 'Patuatuli Road',
+            'osm_building:building': 'yes',
+            'osm_building:building:levels': 4,
+            'osm_building:ctr:lat': '23.707316084046038',
+            'osm_building:ctr:lon': '90.40849938337506',
+            'osm_building:name': 'kol'
+        }
+    ]
 
     def _create_childrens_survey(self, filename="childrens_survey.xls"):
         survey = create_survey_from_xls(_logger_fixture_path(
@@ -2133,3 +2151,147 @@ def test_string_to_date_with_xls_validation(self):
 
         val = ExportBuilder.string_to_date_with_xls_validation(0.4)
         self.assertEqual(val, 0.4)
+
+    def _create_osm_survey(self):
+        # publish form
+        osm_fixtures_dir = os.path.join(settings.PROJECT_ROOT, 'apps', 'api',
+                                        'tests', 'fixtures', 'osm')
+        xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
+        self._publish_xls_file_and_set_xform(xlsform_path)
+
+        # make submissions
+        filenames = [
+            'OSMWay234134797.osm',
+            'OSMWay34298972.osm',
+        ]
+        paths = [os.path.join(osm_fixtures_dir, filename)
+                 for filename in filenames]
+        submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
+        self._make_submission_w_attachment(submission_path, paths)
+        survey = create_survey_from_xls(xlsform_path)
+        return survey
+
+    def test_xls_export_with_osm_data(self):
+        survey = self._create_osm_survey()
+        export_builder = ExportBuilder()
+        export_builder.set_survey(survey)
+        export_builder.set_osm_columns(self.xform)
+        temp_xls_file = NamedTemporaryFile(suffix='.xlsx')
+        export_builder.to_xls_export(temp_xls_file.name, self.osm_data)
+        temp_xls_file.seek(0)
+        wb = load_workbook(temp_xls_file.name)
+        osm_data_sheet = wb["osm"]
+        rows = [row for row in osm_data_sheet.rows]
+        xls_headers = [a.value for a in rows[0]]
+        temp_xls_file.close()
+
+        expected_column_headers = [
+            u'photo', u'osm_road', u'osm_building', u'fav_color',
+            u'form_completed', u'meta/instanceID', u'_id', u'_uuid',
+            u'_submission_time', u'_index', u'_parent_table_name',
+            u'_parent_index', u'_tags', u'_notes', u'_version', u'_duration',
+            u'_submitted_by', u'osm_road:ctr:lat', u'osm_road:ctr:lon',
+            u'osm_road:highway', u'osm_road:lanes', u'osm_road:name',
+            u'osm_road:way:id', u'osm_building:building',
+            u'osm_building:building:levels', u'osm_building:ctr:lat',
+            u'osm_building:ctr:lon', u'osm_building:name',
+            u'osm_building:way:id']
+        self.assertEqual(sorted(expected_column_headers), sorted(xls_headers))
+
+        submission = [a.value for a in rows[1]]
+        self.assertEqual(submission[0], u'1424308569120.jpg')
+        self.assertEqual(submission[17], u'23.708174238006087')
+        self.assertEqual(submission[19], u'tertiary')
+        self.assertEqual(submission[21], u'Patuatuli Road')
+        self.assertEqual(submission[25], u'23.707316084046038')
+        self.assertEqual(submission[27], u'kol')
+
+    def test_zipped_csv_export_with_osm_data(self):
+        survey = self._create_osm_survey()
+        export_builder = ExportBuilder()
+        export_builder.set_survey(survey)
+        export_builder.set_osm_columns(self.xform)
+        temp_zip_file = NamedTemporaryFile(suffix='.zip')
+        export_builder.to_zipped_csv(temp_zip_file.name, self.osm_data)
+        temp_zip_file.seek(0)
+        temp_dir = tempfile.mkdtemp()
+        zip_file = zipfile.ZipFile(temp_zip_file.name, "r")
+        zip_file.extractall(temp_dir)
+        zip_file.close()
+        temp_zip_file.close()
+
+        with open(os.path.join(temp_dir, "osm.csv")) as csv_file:
+            reader = csv.reader(csv_file)
+            rows = [row for row in reader]
+
+            expected_column_headers = [
+                'photo', 'osm_road', 'osm_building', 'fav_color',
+                'form_completed', 'meta/instanceID', '_id', '_uuid',
+                '_submission_time', '_index', '_parent_table_name',
+                '_parent_index', '_tags', '_notes', '_version', '_duration',
+                '_submitted_by', 'osm_road:ctr:lat', 'osm_road:ctr:lon',
+                'osm_road:highway', 'osm_road:lanes', 'osm_road:name',
+                'osm_road:way:id', 'osm_building:building',
+                'osm_building:building:levels', 'osm_building:ctr:lat',
+                'osm_building:ctr:lon', 'osm_building:name',
+                'osm_building:way:id']
+
+            self.assertEqual(sorted(rows[0]), sorted(expected_column_headers))
+            self.assertEqual(rows[1][0], '1424308569120.jpg')
+            self.assertEqual(rows[1][1], 'OSMWay234134797.osm')
+            self.assertEqual(rows[1][17], '23.708174238006087')
+            self.assertEqual(rows[1][19], 'tertiary')
+            self.assertEqual(rows[1][21], 'Patuatuli Road')
+            self.assertEqual(rows[1][27], 'kol')
+
+    def test_zipped_sav_export_with_osm_data(self):
+        survey = self._create_osm_survey()
+        osm_data = [{
+            'photo': '1424308569120.jpg',
+            'osm_road': 'OSMWay234134797.osm',
+            'osm_building': 'OSMWay34298972.osm',
+            'fav_color': 'red',
+            'osm_road_ctr_lat': '23.708174238006087',
+            'osm_road_ctr_lon': '90.40946505581161',
+            'osm_road_highway': 'tertiary',
+            'osm_road_lanes': 2,
+            'osm_road_name': 'Patuatuli Road',
+            'osm_building_building': 'yes',
+            'osm_building_building_levels': 4,
+            'osm_building_ctr_lat': '23.707316084046038',
+            'osm_building_ctr_lon': '90.40849938337506',
+            'osm_building_name': 'kol'
+        }]
+        export_builder = ExportBuilder()
+        export_builder.set_survey(survey)
+        export_builder.set_osm_columns(self.xform)
+        temp_zip_file = NamedTemporaryFile(suffix='.zip')
+        export_builder.to_zipped_sav(temp_zip_file.name, osm_data)
+        temp_zip_file.seek(0)
+        temp_dir = tempfile.mkdtemp()
+        zip_file = zipfile.ZipFile(temp_zip_file.name, "r")
+        zip_file.extractall(temp_dir)
+        zip_file.close()
+        temp_zip_file.close()
+
+        with SavReader(os.path.join(temp_dir, "osm.sav"),
+                       returnHeader=True) as reader:
+            rows = [r for r in reader]
+            expected_column_headers = [
+                'photo', 'osm_road', 'osm_building', 'fav_color',
+                'form_completed', 'meta.instanceID', '@_id', '@_uuid',
+                '@_submission_time', '@_index', '@_parent_table_name',
+                '@_parent_index', '@_tags', '@_notes', '@_version',
+                '@_duration', '@_submitted_by', 'osm_road_ctr_lat',
+                'osm_road_ctr_lon', 'osm_road_highway', 'osm_road_lanes',
+                'osm_road_name', 'osm_road_way_id', 'osm_building_building',
+                'osm_building_building_levels', 'osm_building_ctr_lat',
+                'osm_building_ctr_lon', 'osm_building_name',
+                'osm_building_way_id']
+            self.assertEqual(sorted(rows[0]), sorted(expected_column_headers))
+            self.assertEqual(rows[1][0], '1424308569120.jpg')
+            self.assertEqual(rows[1][1], 'OSMWay234134797.osm')
+            self.assertEqual(rows[1][17], '23.708174238006087')
+            self.assertEqual(rows[1][19], 'tertiary')
+            self.assertEqual(rows[1][21], 'Patuatuli Road')
+            self.assertEqual(rows[1][27], 'kol')

onadata/libs/utils/export_builder.py

@@ -17,6 +17,7 @@
 
 from savReaderWriter import SavWriter
 
+from onadata.apps.logger.models.osmdata import OsmData
 from onadata.apps.logger.models.xform import _encode_for_mongo,\
     QUESTION_TYPES_TO_EXCLUDE
 from onadata.apps.viewer.models.data_dictionary import DataDictionary
@@ -247,6 +248,7 @@ class ExportBuilder(object):
     def __init__(self):
         self.extra_columns = (
             self.EXTRA_FIELDS + getattr(settings, 'EXTRA_COLUMNS', []))
+        self.osm_columns = []
 
     @classmethod
     def string_to_date_with_xls_validation(cls, date_str):
@@ -429,6 +431,14 @@ def _append_xpaths_to_section(current_section_name, field_list, xpath,
             self.select_multiples, self.gps_fields, self.encoded_fields,
             self.GROUP_DELIMITER, self.TRUNCATE_GROUP_TITLE)
 
+    def set_osm_columns(self, xform):
+        osm_fields = self.dd.get_survey_elements_of_type('osm')
+        if osm_fields:
+            for field in osm_fields:
+                self.osm_columns += OsmData.get_tag_keys(
+                    xform, field.get_abbreviated_xpath(),
+                    include_prefix=True)
+
     def section_by_name(self, name):
         matches = filter(lambda s: s['name'] == name, self.sections)
         assert(len(matches) == 1)
@@ -894,8 +904,12 @@ def _is_numeric(xpath, element_type, data_dictionary):
         var_labels = {}
         var_names = []
         fields_and_labels = []
+        osm_columns = [column.replace(':', '_') for column in self.osm_columns]
+
         elements += [{'title': f, "label": f, "xpath": f, 'type': f}
-                     for f in self.extra_columns]
+                     for f in self.extra_columns] + [
+                    {'title': f, "label": f, "xpath": f, 'type': f}
+                    for f in osm_columns]
 
         for element in elements:
             title = element['title']
@@ -1055,8 +1069,9 @@ def get_fields(self, dataview, section, key):
         if dataview:
             return [element[key] for element in section['elements']
                     if element['title'] in dataview.columns]\
-                + self.extra_columns
+                + self.extra_columns + self.osm_columns
 
         else:
             return [element[key] for element in
-                    section['elements']] + self.extra_columns
+                    section['elements']] + self.extra_columns\
+                    + self.osm_columns

onadata/libs/utils/export_tools.py

@@ -222,6 +222,7 @@ def generate_export(export_type, xform, export_id=None, options=None,
         del options['win_excel_utf8']
 
     export_builder.set_survey(xform.survey)
+    export_builder.set_osm_columns(xform)
 
     temp_file = NamedTemporaryFile(suffix=("." + extension))