diff --git a/onadata/apps/viewer/tests/test_export_builder.py b/onadata/apps/viewer/tests/test_export_builder.py index 24a7f3f7a5..e4cf7a0dd4 100644 --- a/onadata/apps/viewer/tests/test_export_builder.py +++ b/onadata/apps/viewer/tests/test_export_builder.py @@ -201,6 +201,24 @@ class TestExportBuilder(TestBase): ] } ] + osm_data = [ + { + 'photo': '1424308569120.jpg', + 'osm_road': 'OSMWay234134797.osm', + 'osm_building': 'OSMWay34298972.osm', + 'fav_color': 'red', + 'osm_road:ctr:lat': '23.708174238006087', + 'osm_road:ctr:lon': '90.40946505581161', + 'osm_road:highway': 'tertiary', + 'osm_road:lanes': 2, + 'osm_road:name': 'Patuatuli Road', + 'osm_building:building': 'yes', + 'osm_building:building:levels': 4, + 'osm_building:ctr:lat': '23.707316084046038', + 'osm_building:ctr:lon': '90.40849938337506', + 'osm_building:name': 'kol' + } + ] def _create_childrens_survey(self, filename="childrens_survey.xls"): survey = create_survey_from_xls(_logger_fixture_path( @@ -2133,3 +2151,147 @@ def test_string_to_date_with_xls_validation(self): val = ExportBuilder.string_to_date_with_xls_validation(0.4) self.assertEqual(val, 0.4) + + def _create_osm_survey(self): + # publish form + osm_fixtures_dir = os.path.join(settings.PROJECT_ROOT, 'apps', 'api', + 'tests', 'fixtures', 'osm') + xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + self._publish_xls_file_and_set_xform(xlsform_path) + + # make submissions + filenames = [ + 'OSMWay234134797.osm', + 'OSMWay34298972.osm', + ] + paths = [os.path.join(osm_fixtures_dir, filename) + for filename in filenames] + submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') + self._make_submission_w_attachment(submission_path, paths) + survey = create_survey_from_xls(xlsform_path) + return survey + + def test_xls_export_with_osm_data(self): + survey = self._create_osm_survey() + export_builder = ExportBuilder() + export_builder.set_survey(survey) + export_builder.set_osm_columns(self.xform) + temp_xls_file = NamedTemporaryFile(suffix='.xlsx') + export_builder.to_xls_export(temp_xls_file.name, self.osm_data) + temp_xls_file.seek(0) + wb = load_workbook(temp_xls_file.name) + osm_data_sheet = wb["osm"] + rows = [row for row in osm_data_sheet.rows] + xls_headers = [a.value for a in rows[0]] + temp_xls_file.close() + + expected_column_headers = [ + u'photo', u'osm_road', u'osm_building', u'fav_color', + u'form_completed', u'meta/instanceID', u'_id', u'_uuid', + u'_submission_time', u'_index', u'_parent_table_name', + u'_parent_index', u'_tags', u'_notes', u'_version', u'_duration', + u'_submitted_by', u'osm_road:ctr:lat', u'osm_road:ctr:lon', + u'osm_road:highway', u'osm_road:lanes', u'osm_road:name', + u'osm_road:way:id', u'osm_building:building', + u'osm_building:building:levels', u'osm_building:ctr:lat', + u'osm_building:ctr:lon', u'osm_building:name', + u'osm_building:way:id'] + self.assertEqual(sorted(expected_column_headers), sorted(xls_headers)) + + submission = [a.value for a in rows[1]] + self.assertEqual(submission[0], u'1424308569120.jpg') + self.assertEqual(submission[17], u'23.708174238006087') + self.assertEqual(submission[19], u'tertiary') + self.assertEqual(submission[21], u'Patuatuli Road') + self.assertEqual(submission[25], u'23.707316084046038') + self.assertEqual(submission[27], u'kol') + + def test_zipped_csv_export_with_osm_data(self): + survey = self._create_osm_survey() + export_builder = ExportBuilder() + export_builder.set_survey(survey) + export_builder.set_osm_columns(self.xform) + temp_zip_file = NamedTemporaryFile(suffix='.zip') + export_builder.to_zipped_csv(temp_zip_file.name, self.osm_data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + zip_file = zipfile.ZipFile(temp_zip_file.name, "r") + zip_file.extractall(temp_dir) + zip_file.close() + temp_zip_file.close() + + with open(os.path.join(temp_dir, "osm.csv")) as csv_file: + reader = csv.reader(csv_file) + rows = [row for row in reader] + + expected_column_headers = [ + 'photo', 'osm_road', 'osm_building', 'fav_color', + 'form_completed', 'meta/instanceID', '_id', '_uuid', + '_submission_time', '_index', '_parent_table_name', + '_parent_index', '_tags', '_notes', '_version', '_duration', + '_submitted_by', 'osm_road:ctr:lat', 'osm_road:ctr:lon', + 'osm_road:highway', 'osm_road:lanes', 'osm_road:name', + 'osm_road:way:id', 'osm_building:building', + 'osm_building:building:levels', 'osm_building:ctr:lat', + 'osm_building:ctr:lon', 'osm_building:name', + 'osm_building:way:id'] + + self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) + self.assertEqual(rows[1][0], '1424308569120.jpg') + self.assertEqual(rows[1][1], 'OSMWay234134797.osm') + self.assertEqual(rows[1][17], '23.708174238006087') + self.assertEqual(rows[1][19], 'tertiary') + self.assertEqual(rows[1][21], 'Patuatuli Road') + self.assertEqual(rows[1][27], 'kol') + + def test_zipped_sav_export_with_osm_data(self): + survey = self._create_osm_survey() + osm_data = [{ + 'photo': '1424308569120.jpg', + 'osm_road': 'OSMWay234134797.osm', + 'osm_building': 'OSMWay34298972.osm', + 'fav_color': 'red', + 'osm_road_ctr_lat': '23.708174238006087', + 'osm_road_ctr_lon': '90.40946505581161', + 'osm_road_highway': 'tertiary', + 'osm_road_lanes': 2, + 'osm_road_name': 'Patuatuli Road', + 'osm_building_building': 'yes', + 'osm_building_building_levels': 4, + 'osm_building_ctr_lat': '23.707316084046038', + 'osm_building_ctr_lon': '90.40849938337506', + 'osm_building_name': 'kol' + }] + export_builder = ExportBuilder() + export_builder.set_survey(survey) + export_builder.set_osm_columns(self.xform) + temp_zip_file = NamedTemporaryFile(suffix='.zip') + export_builder.to_zipped_sav(temp_zip_file.name, osm_data) + temp_zip_file.seek(0) + temp_dir = tempfile.mkdtemp() + zip_file = zipfile.ZipFile(temp_zip_file.name, "r") + zip_file.extractall(temp_dir) + zip_file.close() + temp_zip_file.close() + + with SavReader(os.path.join(temp_dir, "osm.sav"), + returnHeader=True) as reader: + rows = [r for r in reader] + expected_column_headers = [ + 'photo', 'osm_road', 'osm_building', 'fav_color', + 'form_completed', 'meta.instanceID', '@_id', '@_uuid', + '@_submission_time', '@_index', '@_parent_table_name', + '@_parent_index', '@_tags', '@_notes', '@_version', + '@_duration', '@_submitted_by', 'osm_road_ctr_lat', + 'osm_road_ctr_lon', 'osm_road_highway', 'osm_road_lanes', + 'osm_road_name', 'osm_road_way_id', 'osm_building_building', + 'osm_building_building_levels', 'osm_building_ctr_lat', + 'osm_building_ctr_lon', 'osm_building_name', + 'osm_building_way_id'] + self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) + self.assertEqual(rows[1][0], '1424308569120.jpg') + self.assertEqual(rows[1][1], 'OSMWay234134797.osm') + self.assertEqual(rows[1][17], '23.708174238006087') + self.assertEqual(rows[1][19], 'tertiary') + self.assertEqual(rows[1][21], 'Patuatuli Road') + self.assertEqual(rows[1][27], 'kol') diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index 9ba81a6dc1..9e9d9e51b8 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -17,6 +17,7 @@ from savReaderWriter import SavWriter +from onadata.apps.logger.models.osmdata import OsmData from onadata.apps.logger.models.xform import _encode_for_mongo,\ QUESTION_TYPES_TO_EXCLUDE from onadata.apps.viewer.models.data_dictionary import DataDictionary @@ -247,6 +248,7 @@ class ExportBuilder(object): def __init__(self): self.extra_columns = ( self.EXTRA_FIELDS + getattr(settings, 'EXTRA_COLUMNS', [])) + self.osm_columns = [] @classmethod def string_to_date_with_xls_validation(cls, date_str): @@ -429,6 +431,14 @@ def _append_xpaths_to_section(current_section_name, field_list, xpath, self.select_multiples, self.gps_fields, self.encoded_fields, self.GROUP_DELIMITER, self.TRUNCATE_GROUP_TITLE) + def set_osm_columns(self, xform): + osm_fields = self.dd.get_survey_elements_of_type('osm') + if osm_fields: + for field in osm_fields: + self.osm_columns += OsmData.get_tag_keys( + xform, field.get_abbreviated_xpath(), + include_prefix=True) + def section_by_name(self, name): matches = filter(lambda s: s['name'] == name, self.sections) assert(len(matches) == 1) @@ -894,8 +904,12 @@ def _is_numeric(xpath, element_type, data_dictionary): var_labels = {} var_names = [] fields_and_labels = [] + osm_columns = [column.replace(':', '_') for column in self.osm_columns] + elements += [{'title': f, "label": f, "xpath": f, 'type': f} - for f in self.extra_columns] + for f in self.extra_columns] + [ + {'title': f, "label": f, "xpath": f, 'type': f} + for f in osm_columns] for element in elements: title = element['title'] @@ -1055,8 +1069,9 @@ def get_fields(self, dataview, section, key): if dataview: return [element[key] for element in section['elements'] if element['title'] in dataview.columns]\ - + self.extra_columns + + self.extra_columns + self.osm_columns else: return [element[key] for element in - section['elements']] + self.extra_columns + section['elements']] + self.extra_columns\ + + self.osm_columns diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index 1c3cad2db7..1d12612f99 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -222,6 +222,7 @@ def generate_export(export_type, xform, export_id=None, options=None, del options['win_excel_utf8'] export_builder.set_survey(xform.survey) + export_builder.set_osm_columns(xform) temp_file = NamedTemporaryFile(suffix=("." + extension))