diff --git a/docs/metadata.rst b/docs/metadata.rst index 29e554ece7..d9cb178294 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -224,6 +224,27 @@ Link XForm or Dataview as a media example: "url": "https://api.ona.io/api/v1/metadata/7121.json" } + +Link XForm as a GeoJSON media attachment example: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + + curl -X POST -F 'data_type=media' -F 'xform=320' -F 'data_value="xform_geojson 328 places"' -F 'extra_data='{"data_title": "fruits", "data_simple_style": true, "data_geo_field": "geofied_1", "data_fields": "field_1,field_2"}'' https://api.ona.io/api/v1/metadata.json + +:: + + HTTP 201 CREATED + + { + "id": 7121, + "xform": 320, + "data_value": "xform_geojson 328 places", + "data_type": "media", + "extra_data": '{"data_title": "fruits", "data_simple_style": true, "data_geo_field": "geofied_1", "data_fields": "field_1,field_2"}' + "url": "https://api.ona.io/api/v1/metadata/7121.json" + } + Create XForm meta permissions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Set meta permissions for a specific form by passing two roles that are pipe delimited. diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index b4a1e8b7a1..b3092c9bb0 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -467,7 +467,11 @@ def _submit_transport_instance_w_attachment( def _post_metadata(self, data, test=True): count = MetaData.objects.count() view = MetaDataViewSet.as_view({"post": "create"}) - request = self.factory.post("/", data, **self.extra) + request = self.factory.post( + "/", + data=data, + **self.extra, + format='json' if 'extra_data' in data else None) response = view(request) @@ -481,8 +485,23 @@ def _post_metadata(self, data, test=True): return response - def _add_form_metadata(self, xform, data_type, data_value, path=None, test=True): - data = {"data_type": data_type, "data_value": data_value, "xform": xform.id} + def _add_form_metadata( + self, + xform, + data_type, + data_value, + path=None, + test=True, + extra_data=None, + ): + data = { + "data_type": data_type, + "data_value": data_value, + "xform": xform.id + } + + if extra_data: + data.update({"extra_data": extra_data}) if path and data_value: with open(path, "rb") as media_file: diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index b6afb10e2c..dcdf051f5b 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -2045,7 +2045,7 @@ def test_geojson_simple_style_title_prop(self): data_get = { "geo_field": "location", "simple_style": "true", - "title": "shape" + "title": "location" } view = DataViewSet.as_view({"get": "retrieve"}) @@ -2057,7 +2057,7 @@ def test_geojson_simple_style_title_prop(self): self.assertEqual(response.status_code, 200) self.assertIn('title', response.data["properties"]) - self.assertEqual('shape', response.data["properties"]["title"]) + self.assertEqual('-1.294197 36.787219 0 34', response.data["properties"]["title"]) def test_geojson_geofield(self): self._publish_submit_geojson() diff --git a/onadata/apps/api/tests/viewsets/test_metadata_viewset.py b/onadata/apps/api/tests/viewsets/test_metadata_viewset.py index 255c0690a7..40bb3b08a9 100644 --- a/onadata/apps/api/tests/viewsets/test_metadata_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_metadata_viewset.py @@ -163,6 +163,7 @@ def test_get_metadata(self): 'xform': self.xform.pk, 'data_value': u'1335783522563.jpg', 'data_type': u'media', + 'extra_data': None, 'data_file': u'http://localhost:8000/media/%s/formid-media/' '1335783522563.jpg' % self.user.username, 'data_file_type': u'image/jpeg', @@ -271,6 +272,29 @@ def test_add_media_xform_link(self): self.assertEqual(response['Content-Disposition'], 'attachment; filename=transportation.csv') + def test_add_media_geojson_link(self): + data_type = 'media' + data_value = 'xform_geojson {} transportation'.format(self.xform.pk) + extra_data = { + "data_title": "test", + "data_simple_style": True, + "data_geo_field": "test", + "data_fields": "transport/available_transportation_types_to_referral_facility/ambulance" # noqa + } + self._add_form_metadata( + self.xform, + data_type, + data_value, + extra_data=extra_data + ) + self.assertIsNotNone(self.metadata_data['media_url']) + request = self.factory.get('/', **self.extra) + ext = self.data_value[self.data_value.rindex('.') + 1:] + response = self.view(request, pk=self.metadata.pk, format=ext) + self.assertEqual(response.status_code, 200) + self.assertEqual(response['Content-Disposition'], + 'attachment; filename=transportation.geojson') + def test_add_media_dataview_link(self): self._create_dataview() data_type = 'media' diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index e56fbcd73b..065aeb8344 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -557,6 +557,7 @@ def test_view_xls_form(self): ("data_value", "https://enketo.ona.io/::YY8M"), ("data_type", "enketo_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), @@ -571,6 +572,7 @@ def test_view_xls_form(self): ("data_value", "https://enketo.ona.io/preview/::YY8M"), ("data_type", "enketo_preview_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), @@ -588,6 +590,7 @@ def test_view_xls_form(self): ("data_value", "http://enketo.ona.io/single/::XZqoZ94y"), ("data_type", "enketo_single_submit_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 92ee3d9e28..c9291a7dff 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -383,6 +383,7 @@ def test_public_form_list(self): ("data_value", "https://enketo.ona.io/::YY8M"), ("data_type", "enketo_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), @@ -397,6 +398,7 @@ def test_public_form_list(self): ("data_value", "https://enketo.ona.io/preview/::YY8M"), ("data_type", "enketo_preview_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), @@ -414,6 +416,7 @@ def test_public_form_list(self): ("data_value", "http://enketo.ona.io/single/::XZqoZ94y"), ("data_type", "enketo_single_submit_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), @@ -627,6 +630,7 @@ def test_form_get(self): ("data_value", "https://enketo.ona.io/::YY8M"), ("data_type", "enketo_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), @@ -641,6 +645,7 @@ def test_form_get(self): ("data_value", "https://enketo.ona.io/preview/::YY8M"), ("data_type", "enketo_preview_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), @@ -658,6 +663,7 @@ def test_form_get(self): ("data_value", "http://enketo.ona.io/single/::XZqoZ94y"), ("data_type", "enketo_single_submit_url"), ("data_file", None), + ("extra_data", {}), ("data_file_type", None), ("media_url", None), ("file_hash", None), diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index 798e5d21e0..7d27ad3938 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -39,7 +39,6 @@ from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.forms import QuickConverter from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.viewer.models.export import Export from onadata.apps.viewer.models.parsed_instance import datetime_from_str from onadata.libs.baseviewset import DefaultBaseViewset from onadata.libs.models.share_project import ShareProject @@ -56,7 +55,9 @@ get_role_in_org, is_organization, ) -from onadata.libs.utils.api_export_tools import custom_response_handler +from onadata.libs.utils.api_export_tools import ( + custom_response_handler, get_metadata_format +) from onadata.libs.utils.cache_tools import ( PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, @@ -626,14 +627,16 @@ def get_data_value_objects(value): if obj: dataview = obj if isinstance(obj, DataView) else False xform = obj.xform if isinstance(obj, DataView) else obj + export_type = get_metadata_format(metadata.data_value) return custom_response_handler( request, xform, {}, - Export.CSV_EXPORT, + export_type, filename=filename, dataview=dataview, + metadata=metadata, ) return HttpResponseRedirect(metadata.data_value) diff --git a/onadata/apps/api/viewsets/metadata_viewset.py b/onadata/apps/api/viewsets/metadata_viewset.py index 3ffd196f14..0ce33444ec 100644 --- a/onadata/apps/api/viewsets/metadata_viewset.py +++ b/onadata/apps/api/viewsets/metadata_viewset.py @@ -41,7 +41,6 @@ class MetaDataViewSet(AuthenticateHeaderMixin, def retrieve(self, request, *args, **kwargs): self.object = self.get_object() - if isinstance(request.accepted_renderer, MediaFileRenderer) \ and self.object.data_file is not None: diff --git a/onadata/apps/main/migrations/0012_metadata_extra_data.py b/onadata/apps/main/migrations/0012_metadata_extra_data.py new file mode 100644 index 0000000000..f0bdb79fea --- /dev/null +++ b/onadata/apps/main/migrations/0012_metadata_extra_data.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-06-24 11:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0011_auto_20220510_0907'), + ] + + operations = [ + migrations.AddField( + model_name='metadata', + name='extra_data', + field=models.JSONField(default=dict, blank=True, null=True), + ), + ] diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index 025af48010..0ac4e936d0 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -186,6 +186,7 @@ class MetaData(models.Model): data_type = models.CharField(max_length=255) data_value = models.CharField(max_length=255) + extra_data = models.JSONField(default=dict, blank=True, null=True) data_file = models.FileField(upload_to=upload_to, blank=True, null=True) data_file_type = models.CharField(max_length=255, blank=True, null=True) file_hash = models.CharField(max_length=50, blank=True, null=True) diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index 4e28916c31..fc3ea9bf3f 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -77,6 +77,7 @@ class Export(models.Model): EXTERNAL_EXPORT = "external" OSM_EXPORT = OSM GOOGLE_SHEETS_EXPORT = "gsheets" + GEOJSON_EXPORT = "geojson" EXPORT_MIMES = { "xls": "vnd.ms-excel", @@ -87,6 +88,7 @@ class Export(models.Model): "sav_zip": "zip", "sav": "sav", "kml": "vnd.google-earth.kml+xml", + "geojson": "geo+json", OSM: OSM, } @@ -101,6 +103,7 @@ class Export(models.Model): (EXTERNAL_EXPORT, "Excel"), (OSM, OSM), (GOOGLE_SHEETS_EXPORT, "Google Sheets"), + (GEOJSON_EXPORT, "geojson"), ] EXPORT_OPTION_FIELDS = [ diff --git a/onadata/libs/serializers/geojson_serializer.py b/onadata/libs/serializers/geojson_serializer.py index 26b7f70898..e3e9ce9dcb 100644 --- a/onadata/libs/serializers/geojson_serializer.py +++ b/onadata/libs/serializers/geojson_serializer.py @@ -111,12 +111,17 @@ def to_representation(self, obj): ret["properties"][field] = obj.json.get(field) if obj and ret and request: + fields = request.query_params.get("fields") geo_field = request.query_params.get("geo_field") simple_style = request.query_params.get("simple_style") title = request.query_params.get("title") if geo_field: - if "properties" in ret and title: - ret["properties"]["title"] = title + if "properties" in ret: + if title: + ret["properties"]["title"] = obj.json.get(title) + if fields: + for field in fields.split(","): + ret["properties"][field] = obj.json.get(field) points = obj.json.get(geo_field) geometry = ( geometry_from_string(points, simple_style) diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index bf92e52a1a..cf7ca3c67b 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -17,10 +17,13 @@ from rest_framework import serializers from rest_framework.reverse import reverse + from onadata.apps.api.tools import update_role_by_meta_xform_perms +from onadata.libs.utils.api_export_tools import get_metadata_format from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.models import MetaData from onadata.libs.permissions import ROLES, ManagerRole +from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.fields.instance_related_field import InstanceRelatedField from onadata.libs.serializers.fields.project_related_field import ProjectRelatedField from onadata.libs.serializers.fields.xform_related_field import XFormRelatedField @@ -71,6 +74,8 @@ def get_linked_object(parts): """ if isinstance(parts, list) and parts: obj_type = parts[0] + if "geojson" in obj_type: + obj_type = obj_type.split("_")[0] if obj_type in [DATAVIEW_TAG, XFORM_TAG] and len(parts) > 1: obj_pk = parts[1] try: @@ -103,6 +108,7 @@ class MetaDataSerializer(serializers.HyperlinkedModelSerializer): data_value = serializers.CharField(max_length=255, required=True) data_type = serializers.ChoiceField(choices=METADATA_TYPES) data_file = serializers.FileField(required=False) + extra_data = JsonField(required=False) data_file_type = serializers.CharField( max_length=255, required=False, allow_blank=True ) @@ -122,6 +128,7 @@ class Meta: "data_value", "data_type", "data_file", + "extra_data", "data_file_type", "media_url", "file_hash", @@ -140,14 +147,15 @@ def get_media_url(self, obj): ): return obj.data_file.url if obj.data_type in [MEDIA_TYPE] and obj.is_linked_dataset: + request = self.context.get("request") kwargs = { "kwargs": { "pk": obj.content_object.pk, "username": obj.content_object.user.username, "metadata": obj.pk, }, - "request": self.context.get("request"), - "format": "csv", + "request": request, + "format": get_metadata_format(obj.data_value), } return reverse("xform-media", **kwargs) @@ -262,6 +270,7 @@ def create(self, validated_data): data_type = validated_data.get("data_type") data_file = validated_data.get("data_file") data_file_type = validated_data.get("data_file_type") + extra_data = validated_data.get('extra_data') content_object = self.get_content_object(validated_data) data_value = data_file.name if data_file else validated_data.get("data_value") @@ -303,6 +312,7 @@ def create(self, validated_data): data_type=data_type, data_value=data_value, data_file=data_file, + extra_data=extra_data, data_file_type=data_file_type, object_id=content_object.id, ) diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index 41dac1a253..6bcb4789e6 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -22,6 +22,7 @@ from rest_framework import serializers from rest_framework.reverse import reverse +from onadata.libs.utils.api_export_tools import get_metadata_format from onadata.apps.logger.models import DataView, Instance, XForm, XFormVersion from onadata.apps.main.models.meta_data import MetaData from onadata.libs.exceptions import EnketoError @@ -627,11 +628,12 @@ def get_url(self, obj): "metadata": obj.pk, } request = self.context.get("request") + extension = get_metadata_format(obj.data_value) try: fmt_index = obj.data_value.rindex(".") + 1 fmt = obj.data_value[fmt_index:] except ValueError: - fmt = "csv" + fmt = extension url = reverse("xform-media", kwargs=kwargs, request=request, format=fmt.lower()) group_delimiter = self.context.get(GROUP_DELIMETER_TAG) @@ -689,10 +691,11 @@ def get_filename(self, obj): """ filename = obj.data_value parts = filename.split(" ") + extension = get_metadata_format(filename) # filtered dataset is of the form "xform PK name", filename is the # third item if len(parts) > 2: - filename = f"{parts[2]}.csv" + filename = f"{parts[2]}.{extension}" else: try: URLValidator()(filename) diff --git a/onadata/libs/tests/utils/test_api_export_tools.py b/onadata/libs/tests/utils/test_api_export_tools.py index 2fd263e523..2a4514ed15 100644 --- a/onadata/libs/tests/utils/test_api_export_tools.py +++ b/onadata/libs/tests/utils/test_api_export_tools.py @@ -16,7 +16,10 @@ from onadata.apps.viewer.models.export import Export, ExportConnectionError from onadata.libs.exceptions import ServiceUnavailable from onadata.libs.utils.api_export_tools import ( - get_async_response, process_async_export, response_for_format) + get_async_response, process_async_export, + response_for_format, + get_metadata_format +) from onadata.libs.utils.async_status import SUCCESSFUL, status_msg @@ -145,6 +148,25 @@ def test_response_for_format(self): with self.assertRaises(Http404): response_for_format(xform, 'xls') + def test_get_metadata_format(self): + """ + Test metadata export format/ext. + """ + self._publish_xlsx_file() + xform = XForm.objects.filter().last() + data_value = "xform_geojson {} {}".format( + xform.pk, xform.id_string) + fmt = get_metadata_format(data_value) + self.assertEqual("geojson", fmt) + data_value = "dataview_geojson {} {}".format( + xform.pk, xform.id_string) + fmt = get_metadata_format(data_value) + self.assertEqual("geojson", fmt) + data_value = "xform {} {}".format( + xform.pk, xform.id_string) + fmt = get_metadata_format(data_value) + self.assertEqual(fmt, "csv") + # pylint: disable=invalid-name @mock.patch( 'onadata.libs.utils.api_export_tools.viewer_task.create_async_export') diff --git a/onadata/libs/tests/utils/test_export_tools.py b/onadata/libs/tests/utils/test_export_tools.py index 7f740eafa8..7e30c9554b 100644 --- a/onadata/libs/tests/utils/test_export_tools.py +++ b/onadata/libs/tests/utils/test_export_tools.py @@ -3,6 +3,7 @@ Test export_tools module """ import os +import json import shutil import tempfile import zipfile @@ -22,12 +23,15 @@ from savReaderWriter import SavWriter from onadata.apps.api import tests as api_tests +from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( + TestAbstractViewSet) from onadata.apps.logger.models import Attachment, Instance, XForm from onadata.apps.main.tests.test_base import TestBase from onadata.apps.viewer.models.export import Export from onadata.apps.viewer.models.parsed_instance import query_data from onadata.apps.api.viewsets.data_viewset import DataViewSet from onadata.libs.serializers.merged_xform_serializer import MergedXFormSerializer +from onadata.libs.serializers.xform_serializer import XFormSerializer from onadata.libs.utils.export_builder import encode_if_str, get_value_or_attachment_uri from onadata.libs.utils.export_tools import ( ExportBuilder, @@ -36,6 +40,7 @@ generate_export, generate_kml_export, generate_osm_export, + generate_geojson_export, get_repeat_index_tags, kml_export_data, parse_request_export_options, @@ -48,7 +53,7 @@ def _logger_fixture_path(*args): return os.path.join(settings.PROJECT_ROOT, "libs", "tests", "fixtures", *args) -class TestExportTools(TestBase): +class TestExportTools(TestBase, TestAbstractViewSet): """ Test export_tools functions. """ @@ -532,6 +537,105 @@ def test_kml_exports(self): self.assertIsNotNone(export) self.assertTrue(export.is_successful) + def test_geojson_exports(self): + """ + Test generate_geojson_export() + """ + self.extra = {"HTTP_AUTHORIZATION": f"Token {self.user.auth_token}"} + geo_md = """ + | survey | + | | type | name | label | + | | geopoint | gps | GPS | + | | select one fruits | fruit | Fruit | + + | choices | + | | list name | name | label | + | | fruits | orange | Orange | + | | fruits | mango | Mango | + """ + xform1 = self._publish_markdown(geo_md, self.user, id_string="a") + xml = '-1.28 36.83 0 0orange' + Instance(xform=xform1, xml=xml).save() + request = self.factory.get('/', **self.extra) + XFormSerializer(xform1, context={'request': request}).data + xform1 = XForm.objects.get(id_string="a") + export_type = "geojson" + options = { + "extension": "geojson", + } + self._publish_transportation_form_and_submit_instance() + # set metadata to xform + data_type = "media" + data_value = 'xform_geojson {} {}'.format(xform1.pk, xform1.id_string) + extra_data = { + "data_title": "fruit", + "data_geo_field": "gps", + "data_simple_style": True, + "data_fields": "fruit,gps" + } + response = self._add_form_metadata( + self.xform, data_type, data_value, extra_data=extra_data) + self.assertEqual(response.status_code, 201) + username = self.xform.user.username + id_string = self.xform.id_string + # get metadata instance and pass to geojson export util function + self.assertEqual(self.xform.metadata_set.count(), 1) + metadata = self.xform.metadata_set.all()[0] + export = generate_geojson_export( + export_type, + username, + id_string, + metadata, + options=options, + xform=xform1 + ) + self.assertIsNotNone(export) + self.assertTrue(export.is_successful) + with default_storage.open(export.filepath) as f2: + content = f2.read().decode('utf-8') + geojson = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 36.83, + -1.28 + ] + }, + "properties": { + "fruit": "orange", + "gps": "-1.28 36.83 0 0", + "title": "orange" + } + } + ] + } + content = json.loads(content) + # remove xform and id from properties because they keep changing + del content["features"][0]["properties"]["id"] + del content["features"][0]["properties"]["xform"] + self.assertEqual(content, geojson) + + export_id = export.id + + export.delete() + + export = generate_geojson_export( + export_type, + username, + id_string, + metadata, + export_id=export_id, + options=options, + xform=xform1 + ) + + self.assertIsNotNone(export) + self.assertTrue(export.is_successful) + def test_str_to_bool(self): self.assertTrue(str_to_bool(True)) self.assertTrue(str_to_bool("True")) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index ab0f0d5c0c..719c64eb3f 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -58,6 +58,7 @@ generate_external_export, generate_kml_export, generate_osm_export, + generate_geojson_export, newest_export_for, parse_request_export_options, should_create_new_export, @@ -79,9 +80,21 @@ "zip": Export.ZIP_EXPORT, OSM: Export.OSM_EXPORT, "gsheets": Export.GOOGLE_SHEETS_EXPORT, + "geojson": Export.GEOJSON_EXPORT, } +def get_metadata_format(data_value): + """Returns metadata format/extension""" + fmt = "csv" + + if data_value.startswith("xform_geojson") or data_value.startswith( + "dataview_geojson" + ): + fmt = "geojson" + return fmt + + def include_hxl_row(dv_columns, hxl_columns): """ This function returns a boolean value. If the dataview's columns are not @@ -117,6 +130,7 @@ def custom_response_handler( # noqa: C0901 meta=None, dataview=False, filename=None, + metadata=None, ): """ Returns a HTTP response with export file for download. @@ -174,7 +188,8 @@ def custom_response_handler( # noqa: C0901 # we always re-generate if a filter is specified def _new_export(): return _generate_new_export( - request, xform, query, export_type, dataview_pk=dataview_pk + request, xform, query, export_type, + dataview_pk=dataview_pk, metadata=metadata ) if should_create_new_export(xform, export_type, options, request=request): @@ -217,7 +232,7 @@ def _new_export(): def _generate_new_export( # noqa: C0901 - request, xform, query, export_type, dataview_pk=False + request, xform, query, export_type, dataview_pk=False, metadata=None ): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) @@ -275,6 +290,16 @@ def _generate_new_export( # noqa: C0901 options, xform=xform, ) + elif export_type == Export.GEOJSON_EXPORT: + export = generate_geojson_export( + export_type, + xform.user.username, + xform.id_string, + metadata, + None, + options, + xform=xform, + ) else: options.update(parse_request_export_options(request.query_params)) diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index 55c30cdf34..6cc125c5be 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -10,6 +10,7 @@ import re import sys from datetime import datetime, timedelta +from django.http import HttpRequest from django.conf import settings from django.contrib.auth import get_user_model @@ -24,6 +25,7 @@ import six from json2xlsclient.client import Client from multidb.pinning import use_master + from rest_framework import exceptions from six import iteritems from six.moves.urllib.parse import urlparse @@ -39,6 +41,7 @@ from onadata.apps.viewer.models.export import Export, get_export_options_query_kwargs from onadata.apps.viewer.models.parsed_instance import query_data from onadata.libs.exceptions import J2XException, NoRecordsFoundError +from onadata.libs.serializers.geojson_serializer import GeoJsonSerializer from onadata.libs.utils.common_tags import DATAVIEW_EXPORT, GROUPNAME_REMOVED_FLAG from onadata.libs.utils.common_tools import ( cmp_to_key, @@ -546,6 +549,60 @@ def generate_kml_export( return export +def generate_geojson_export( + export_type, + username, + id_string, + metadata=None, + export_id=None, + options=None, + xform=None +): + """ + Generates Linked Geojson export + + :param export_type: type of export + :param username: logged in username + :param id_string: xform id_string + :param export_id: ID of export object associated with the request + :param options: additional parameters required for the lookup. + :param ext: File extension of the generated export + """ + + extension = options.get("extension", export_type) + if xform is None: + xform = XForm.objects.get(user__username=username, id_string=id_string) + request = HttpRequest() + extra_data = metadata.extra_data + # build out query params to be used in GeoJsonSerializer + request.query_params = { + "geo_field": extra_data.get("data_geo_field"), + "simple_style": extra_data.get("data_simple_style"), + "title": extra_data.get("data_title"), + "fields": extra_data.get("data_fields") + } + _context = {} + _context['request'] = request + content = GeoJsonSerializer(xform.instances.all(), many=True, context=_context) + data_to_write = json.dumps(content.data).encode('utf-8') + timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + basename = f"{id_string}_{timestamp}" + filename = basename + "." + extension + file_path = os.path.join(username, "exports", id_string, export_type, filename) + + export_filename = write_temp_file_to_path(extension, data_to_write, file_path) + + export = get_or_create_export_object(export_id, options, xform, export_type) + + dir_name, basename = os.path.split(export_filename) + export.filedir = dir_name + export.filename = basename + export.internal_status = Export.SUCCESSFUL + export.save() + + return export + + def kml_export_data(id_string, user, xform=None): """ KML export data from form submissions.