Migrate qgis_server code to geonode 2.6.x (#164)

Add qgis_server backend support for geonode:

* Add QGIS Server app.

* Push layers to local storage for qgis backend.

* Handling for updating old layer object.

* Implementation for deleting a layer.

* Put delete layer in pre_delete QGISServerLayer

* Add link to download zip file.

* Better link in layer detail page.

* create a QGIS project when a layer is uploaded

* update qgis project

* add gis tools

* serve tiles with a WMS wrapper

* forward legend request to QGIS WMS if the legend is not in cache

* Add tile url to link. Able to show the layer now.

* Add link for legend.

* check if the PNG is valid, remove it if it is not valid

* get a thumbnail from qgis server

* Move import signals to end of model to avoid circular import.

* Add thumbnail for layers.

* add indicator for analysis task process

* Hide filename for generated qml legend

* First working wms map in the map page creation.

* better code structure and add local_setting sample.

* enable WFS in the qgis template

* Replace SRID 900913 to 3857.

* Add DescribeLayer in qgisserver app.

* Add DescribeFeatureType in qgisserver app.

* Add Describe FeatureType

* Renaming wms to better method name.

* Styling legend.

* Make print map dialog works.

* Temporary hack for map printing.

* fix transparency when calling the WMS

* fix qgis project with a raster

* fix the projection in the QGIS template if different than 4326

* remove cache when we remove the qgis layer

* set the attributes for a vector layer using WFS

* remove qgis project template

* fix attribute table when calling the WFS with the new server plugin

* Set local layer.

* Call the server plugin while saving the layer in the DB to create the qgis project

* update mimetype to content_type due to django 1.7

* create qgis map

* Refactor django.conf.settings access

* Refactor thumbnail creation to use celery

* Use relative url for layer thumbnail.

* Update migrations for geonode:2.6.x

* fix #19: thumbnail and legend url resolver

* issue #21: Provide GeoTIFF download url

* delete and copy layers properly:

* fix #24: File saved as different name sometimes

* fix #25: Cache files not deleted when layer deleted

geonode/qgis_server/__init__.py

@@ -0,0 +1,8 @@
+__author__ = 'ismailsunni'
+__project_name__ = 'geonode'
+__filename__ = '__init__.py'
+__date__ = '1/19/16'
+__copyright__ = '[email protected]'
+
+
+__version__ = '0.1.1'

geonode/qgis_server/admin.py

@@ -0,0 +1,19 @@
+__author__ = 'ismailsunni'
+__project_name__ = 'geonode'
+__filename__ = 'admin'
+__date__ = '1/28/16'
+__copyright__ = '[email protected]'
+
+
+from django.contrib import admin
+from geonode_qgis_server.models import QGISServerLayer
+
+
+# Register your models here.
+class QGISServerLayerAdmin(admin.ModelAdmin):
+    list_display = [
+        'layer',
+        'base_layer_path'
+    ]
+
+admin.site.register(QGISServerLayer, QGISServerLayerAdmin)

geonode/qgis_server/gis_tools.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+
+import logging
+from urllib import urlencode, urlretrieve
+from os.path import splitext
+from math import atan, degrees, sinh, pi
+from lxml import etree
+
+from django.conf import settings as geonode_config
+from geonode.layers.models import Attribute
+from geonode_qgis_server.models import QGISServerLayer
+
+logger = logging.getLogger(__name__)
+
+
+def set_attributes(layer, overwrite=False):
+    """Retrieve layer attribute names & types from QGIS Server using
+    the WFS/WMS/WCS, then store in GeoNode database using Attribute model.
+
+    This function is copied/adapted from set_attributes in geoserver/helper.py.
+
+    :param layer: The layer where we want to add/update the attributes.
+    :type layer: QGISServerLayer
+
+    :param overwrite: If we should overwrite or not in the database.
+    :type overwrite: bool
+    """
+    if layer.storeType in ['dataStore']:
+        layer_name = layer.typename.encode('utf-8')
+        qgis_layer = QGISServerLayer.objects.get(layer=layer)
+
+        qgis_server = geonode_config.QGIS_SERVER_CONFIG['qgis_server_url']
+        basename, _ = splitext(qgis_layer.base_layer_path)
+        dft_url = qgis_server + '?' + urlencode({
+            'MAP': basename + '.qgs',
+            'SERVICE': 'WFS',
+            'VERSION': '1.0.0',
+            'REQUEST': 'DescribeFeatureType',
+            'LAYER': layer_name
+        })
+
+        # noinspection PyBroadException
+        try:
+            temp_file = urlretrieve(dft_url)[0]
+            with open (temp_file, 'r') as wfs_file:
+                doc = etree.fromstring(wfs_file.read())
+
+            path = './/{xsd}extension/{xsd}sequence/{xsd}element'.format(
+                xsd='{http://www.w3.org/2001/XMLSchema}')
+
+            attribute_map = [
+                [n.attrib['name'], n.attrib['type']] for n in doc.findall(
+                    path) if n.attrib.get('name') and n.attrib.get('type')]
+
+        except:
+            attribute_map = []
+    else:
+        attribute_map = []
+
+    # We need 3 more items for description, attribute_label and display_order
+    attribute_map_dict = {
+        'field': 0,
+        'ftype': 1,
+        'description': 2,
+        'label': 3,
+        'display_order': 4,
+    }
+    for attribute in attribute_map:
+        attribute.extend((None, None, 0))
+
+    attributes = layer.attribute_set.all()
+
+    # Delete existing attributes if they no longer exist in an updated layer.
+    for la in attributes:
+        lafound = False
+        for attribute in attribute_map:
+            field, ftype, description, label, display_order = attribute
+            if field == la.attribute:
+                lafound = True
+                # store description and attribute_label in attribute_map
+                attribute[attribute_map_dict['description']] = la.description
+                attribute[attribute_map_dict['label']] = la.attribute_label
+                attribute[attribute_map_dict['display_order']] = la.display_order
+
+        if overwrite or not lafound:
+            logger.debug(
+                'Going to delete [%s] for [%s]',
+                la.attribute,
+                layer.name.encode('utf-8'))
+            la.delete()
+
+    # Add new layer attributes if they don't already exist.
+    if attribute_map is not None:
+        iter = len(Attribute.objects.filter(layer=layer)) + 1
+        for attribute in attribute_map:
+            field, ftype, description, label, display_order = attribute
+            if field is not None:
+                la, created = Attribute.objects.get_or_create(
+                    layer=layer, attribute=field, attribute_type=ftype,
+                    description=description, attribute_label=label,
+                    display_order=display_order)
+                if created:
+                    la.visible = ftype.find('gml:') != 0
+                    la.display_order = iter
+                    la.save()
+                    iter += 1
+                    logger.debug(
+                        'Created [%s] attribute for [%s]',
+                        field,
+                        layer.name.encode('utf-8'))
+    else:
+        logger.debug('No attributes found')
+
+
+def num2deg(x_tile, y_tile, zoom):
+    """Conversion of X,Y and zoom from a TMS url to lat/lon coordinates
+    See http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames
+
+    :param x_tile: The X tile number.
+    :type x_tile: integer
+
+    :param y_tile: The Y tile number.
+    :type y_tile: integer
+
+    :param zoom: The zoom level, usually between 0 and 20.
+    :type zoom: integer
+
+    :return: Tuple (lat,lon).
+    :rtype: list
+    """
+    n = 2.0 ** zoom
+    lon_deg = x_tile / n * 360.0 - 180.0
+    lat_rad = atan(sinh(pi * (1 - 2 * y_tile / n)))
+    lat_deg = degrees(lat_rad)
+    return lat_deg, lon_deg

geonode/qgis_server/migrations/0001_initial.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layers', '24_to_26'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='QGISServerLayer',
+            fields=[
+                ('layer', models.OneToOneField(primary_key=True, serialize=False, to='layers.Layer')),
+                ('base_layer_path', models.CharField(help_text=b'Location of the base layer.', max_length=100, verbose_name=b'Base Layer Path')),
+            ],
+        ),
+    ]

geonode/qgis_server/models.py

@@ -0,0 +1,63 @@
+import os
+import logging
+from shutil import rmtree
+from django.db import models
+from django.conf import settings
+
+from geonode.layers.models import Layer
+from geonode.maps.models import MapLayer
+
+logger = logging.getLogger("geonode_qgis_server.models")
+
+QGIS_LAYER_DIRECTORY = settings.QGIS_SERVER_CONFIG['layer_directory']
+QGIS_TILES_DIRECTORY = settings.QGIS_SERVER_CONFIG['tiles_directory']
+
+if not os.path.exists(QGIS_LAYER_DIRECTORY):
+    os.mkdir(QGIS_LAYER_DIRECTORY)
+
+
+class QGISServerLayer(models.Model):
+    """Model for Layer in QGIS Server Backend.
+    """
+
+    accepted_format = [
+        'tif', 'tiff', 'asc', 'shp', 'shx', 'dbf', 'prj', 'qml', 'xml', 'qgs']
+
+    geotiff_format = ['tif', 'tiff']
+
+    layer = models.OneToOneField(
+        Layer,
+        primary_key=True,
+        name='layer'
+    )
+    base_layer_path = models.CharField(
+        name='base_layer_path',
+        verbose_name='Base Layer Path',
+        help_text='Location of the base layer.',
+        max_length=100
+    )
+
+    def delete_qgis_layer(self):
+        """Delete all files related to this object from disk."""
+        try:
+            base_path = self.base_layer_path
+            base_name, _ = os.path.splitext(base_path)
+            for ext in QGISServerLayer.accepted_format:
+                file_path = base_name + '.' + ext
+                if os.path.exists(file_path):
+                    os.remove(file_path)
+        except QGISServerLayer.DoesNotExist:
+            logger.debug('QGIS Server Layer not found. Not deleting.')
+            pass
+
+        # Removing the cache.
+        basename, _ = os.path.splitext(self.base_layer_path)
+        basename = os.path.basename(basename)
+        path = os.path.join(QGIS_TILES_DIRECTORY, basename)
+        logger.info('Removing the cache from a qgis layer : %s' % path)
+        try:
+            rmtree(path)
+        except OSError:
+            pass
+
+import geonode_qgis_server.signals