From 704f5422e8321cf81ef0e9678aade9a0326114db Mon Sep 17 00:00:00 2001 From: James Bannister Date: Wed, 1 May 2024 14:54:44 +0100 Subject: [PATCH 01/16] Newest changes --- application/routers/tiles_.py | 161 +++++++++++++++++++++++++++++++ tests/unit/routers/test_tiles.py | 119 +++++++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 application/routers/tiles_.py create mode 100644 tests/unit/routers/test_tiles.py diff --git a/application/routers/tiles_.py b/application/routers/tiles_.py new file mode 100644 index 00000000..5e23bff3 --- /dev/null +++ b/application/routers/tiles_.py @@ -0,0 +1,161 @@ +import logging + +from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse + +import psycopg2 +from io import BytesIO + +from application.settings import get_settings + +router = APIRouter() +logger = logging.getLogger(__name__) + +DATABASE = {"user": "", "password": "", "host": "", "port": "5432", "database": ""} + +DATABASE_CONNECTION = None + +QUERY_PARAMS = { + "table1": "entity t1", + "srid": "4326", + "geomColumn": "t1.geometry", + "attrColumns": "t1.entity, t1.name, t1.reference", +} + + +# ============================================================ +# Helper Funcs +# ============================================================ +def get_db_connection(): + conn_str = get_settings() + + DATABASE["user"] = conn_str.READ_DATABASE_URL.user + DATABASE["password"] = conn_str.READ_DATABASE_URL.password + DATABASE["host"] = conn_str.READ_DATABASE_URL.host + DATABASE["database"] = conn_str.READ_DATABASE_URL.path.split("/")[1] + + +get_db_connection() + + +# Do the tile x/y coordinates make sense at this zoom level? +def tile_is_valid(tile): + if not ("x" in tile and "y" in tile and "zoom" in tile): + return False + + if "format" not in tile or tile["format"] not in ["pbf", "mvt"]: + return False + + size = 2 ** tile["zoom"] + + if tile["x"] >= size or tile["y"] >= size: + return False + + if tile["x"] < 0 or tile["y"] < 0: + return False + + return True + + +def build_db_query(tile): + qry_params = QUERY_PARAMS.copy() + qry_params["dataset"] = tile["dataset"] + qry_params["x"] = tile["x"] + qry_params["y"] = tile["y"] + qry_params["z"] = tile["zoom"] + + query = """ + WITH + webmercator(envelope) AS ( + SELECT ST_TileEnvelope({z}, {x}, {y}) + ), + wgs84(envelope) AS ( + SELECT ST_Transform((SELECT envelope FROM webmercator), {srid}) + ), + b(bounds) AS ( + SELECT ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, {srid}) + ), + geometries(entity, name, reference, wkb_geometry) AS ( + SELECT + {attrColumns}, + CASE WHEN ST_Covers(b.bounds, {geomColumn}) + THEN ST_Transform({geomColumn},{srid}) + ELSE ST_Transform(ST_Intersection(b.bounds, {geomColumn}),{srid}) + END + FROM + {table1} + CROSS JOIN + b + WHERE + {geomColumn} && (SELECT envelope FROM wgs84) + AND + t1.dataset = '{dataset}' + ) + SELECT + ST_AsMVT(tile, '{dataset}') as mvt + FROM ( + SELECT + entity, + name, + reference, + ST_AsMVTGeom(wkb_geometry, (SELECT envelope FROM wgs84)) + FROM geometries + ) AS tile + """.format( + **qry_params + ) + + return query + + +def sql_to_pbf(sql): + global DATABASE_CONNECTION + + # Make and hold connection to database + if not DATABASE_CONNECTION: + try: + DATABASE_CONNECTION = psycopg2.connect(**DATABASE) + except (Exception, psycopg2.Error) as error: + logger.warning(error) + return None + + # Query for MVT + with DATABASE_CONNECTION.cursor() as cur: + cur.execute(sql) + if not cur: + logger.warning(f"sql query failed: {sql}") + return None + + return cur.fetchone()[0] + + return None + + +# ============================================================ +# API Endpoints +# ============================================================ + + +@router.get("/{dataset}/{z}/{x}/{y}.vector.{fmt}") +async def read_tiles_from_postgres(dataset: str, z: int, x: int, y: int, fmt: str): + tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} + + if not tile_is_valid(tile): + raise HTTPException(status_code=400, detail=f"invalid tile path: {tile}") + + sql = build_db_query(tile) + + pbf = sql_to_pbf(sql) + + pbf_buffer = BytesIO() + pbf_buffer.write(pbf) + pbf_buffer.seek(0) + + resp_headers = { + "Access-Control-Allow-Origin": "*", + "Content-Type": "application/vnd.mapbox-vector-tile", + } + + return StreamingResponse( + pbf_buffer, media_type="vnd.mapbox-vector-tile", headers=resp_headers + ) diff --git a/tests/unit/routers/test_tiles.py b/tests/unit/routers/test_tiles.py new file mode 100644 index 00000000..8e1297d6 --- /dev/null +++ b/tests/unit/routers/test_tiles.py @@ -0,0 +1,119 @@ +from unittest.mock import MagicMock, patch +import pytest +from fastapi import HTTPException +from fastapi.responses import StreamingResponse + +from application.routers.tiles_ import ( + read_tiles_from_postgres, + tile_is_valid, + build_db_query, + sql_to_pbf, +) + +# Constants for Testing +VALID_TILE_INFO = { + "x": 512, + "y": 512, + "zoom": 10, + "format": "pbf", + "dataset": "example-dataset", +} +INVALID_TILE_INFO = { + "x": -1, + "y": 512, + "zoom": 10, + "format": "jpg", + "dataset": "example-dataset", +} + + +@pytest.fixture +def valid_tile(): + return VALID_TILE_INFO.copy() + + +@pytest.fixture +def invalid_tile(): + return INVALID_TILE_INFO.copy() + + +@pytest.fixture +def mock_build_db_query(): + with patch("application.routers.tiles_.build_db_query") as mock: + yield mock + + +@pytest.fixture +def mock_sql_to_pbf(): + with patch("application.routers.tiles_.sql_to_pbf") as mock: + mock.return_value = b"sample_pbf_data" + yield mock + + +def test_tile_is_valid(valid_tile): + assert tile_is_valid(valid_tile), "Tile should be valid with correct parameters" + + +def test_tile_is_invalid(invalid_tile): + assert not tile_is_valid( + invalid_tile + ), "Tile should be invalid with incorrect parameters" + + +def test_build_db_query(valid_tile): + query = build_db_query(valid_tile) + assert ( + "SELECT" in query and "FROM" in query + ), "SQL query should be properly formed with SELECT and FROM clauses" + + +@patch("application.routers.tiles_.psycopg2.connect") +def test_sql_to_pbf(mock_connect, valid_tile): + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_connect.return_value = mock_conn + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_cursor.fetchone.return_value = [b"test_pbf_data"] + + sql = build_db_query(valid_tile) + pbf_data = sql_to_pbf(sql) + + assert pbf_data == b"test_pbf_data", "Should return binary PBF data" + mock_cursor.execute.assert_called_with(sql) + mock_cursor.fetchone.assert_called_once() + + +@pytest.mark.asyncio +async def test_read_tiles_from_postgres_invalid_tile(invalid_tile): + with pytest.raises(HTTPException) as excinfo: + await read_tiles_from_postgres( + invalid_tile["dataset"], + invalid_tile["zoom"], + invalid_tile["x"], + invalid_tile["y"], + invalid_tile["format"], + ) + assert ( + excinfo.value.status_code == 400 + ), "Should raise HTTP 400 for invalid tile parameters" + + +@pytest.mark.asyncio +async def test_read_tiles_from_postgres_valid_tile( + mock_build_db_query, mock_sql_to_pbf, valid_tile +): + mock_build_db_query.return_value = "SELECT * FROM tiles" + response = await read_tiles_from_postgres( + valid_tile["dataset"], + valid_tile["zoom"], + valid_tile["x"], + valid_tile["y"], + valid_tile["format"], + ) + + assert isinstance(response, StreamingResponse), "Should return a StreamingResponse" + assert ( + response.status_code == 200 + ), "Response status should be 200 for valid requests" + mock_build_db_query.assert_called_once_with(valid_tile) + mock_sql_to_pbf.assert_called_once_with("SELECT * FROM tiles") From 96c2295af0cf6dbb9ab60bb6a44b693a1c0941c6 Mon Sep 17 00:00:00 2001 From: James Bannister Date: Thu, 2 May 2024 11:47:49 +0100 Subject: [PATCH 02/16] Map Controller changes --- assets/javascripts/MapController.js | 677 +++++++++++++++------------- 1 file changed, 369 insertions(+), 308 deletions(-) diff --git a/assets/javascripts/MapController.js b/assets/javascripts/MapController.js index f4728470..2fb73401 100644 --- a/assets/javascripts/MapController.js +++ b/assets/javascripts/MapController.js @@ -4,7 +4,7 @@ import LayerControls from "./LayerControls.js"; import TiltControl from "./TiltControl.js"; import { capitalizeFirstLetter, preventScroll } from "./utils.js"; import { getApiToken, getFreshApiToken } from "./osApiToken.js"; -import {defaultPaintOptions} from "./defaultPaintOptions.js"; +import { defaultPaintOptions } from "./defaultPaintOptions.js"; export default class MapController { constructor(params) { @@ -20,40 +20,51 @@ export default class MapController { setParams(params) { params = params || {}; - this.mapId = params.mapId || 'mapid'; - this.mapContainerSelector = params.mapContainerSelector || '.dl-map__wrapper'; + this.mapId = params.mapId || "mapid"; + this.mapContainerSelector = + params.mapContainerSelector || ".dl-map__wrapper"; this.vectorTileSources = params.vectorTileSources || []; - this.datasetVectorUrl = params.datasetVectorUrl || null; + this.datasetVectorUrl = + params.datasetVectorUrl || "http://"; + this.apiKey = params.apiKey || null; this.datasets = params.datasets || null; this.minMapZoom = params.minMapZoom || 5; this.maxMapZoom = params.maxMapZoom || 15; - this.baseURL = params.baseURL || 'https://digital-land.github.io'; - this.baseTileStyleFilePath = params.baseTileStyleFilePath || '/static/javascripts/base-tile.json'; - this.popupWidth = params.popupWidth || '260px'; + this.baseURL = params.baseURL || "https://digital-land.github.io"; + this.baseTileStyleFilePath = + params.baseTileStyleFilePath || "/static/javascripts/base-tile.json"; + this.popupWidth = params.popupWidth || "260px"; this.popupMaxListLength = params.popupMaxListLength || 10; - this.LayerControlOptions = params.LayerControlOptions || {enabled: false}; - this.ZoomControlsOptions = params.ZoomControlsOptions || {enabled: false}; - this.FullscreenControl = params.FullscreenControl || {enabled: false}; + this.LayerControlOptions = params.LayerControlOptions || { enabled: false }; + this.ZoomControlsOptions = params.ZoomControlsOptions || { enabled: false }; + this.FullscreenControl = params.FullscreenControl || { enabled: false }; this.geojsons = params.geojsons || []; - this.images = params.images || [{src: '/static/images/location-pointer-sdf-256.png', name: 'custom-marker-256', size: 256}]; - this.paint_options = params.paint_options || null; - this.customStyleJson = '/static/javascripts/OS_VTS_3857_3D.json'; - this.customStyleLayersToBringToFront = [ - 'OS/Names/National/Country', + this.images = params.images || [ + { + src: "/static/images/location-pointer-sdf-256.png", + name: "custom-marker-256", + size: 256, + }, ]; + this.paint_options = params.paint_options || null; + this.customStyleJson = "/static/javascripts/OS_VTS_3857_3D.json"; + this.customStyleLayersToBringToFront = ["OS/Names/National/Country"]; this.useOAuth2 = params.useOAuth2 || false; this.layers = params.layers || []; this.featuresHoveringOver = 0; } getViewFromUrl() { - const urlObj = new URL(document.location) - const hash = urlObj.hash - if(hash){ - const [lat, lng, zoom] = hash.substring(1).split(',') - return {centre: [parseFloat(lng), parseFloat(lat)], zoom: parseFloat(zoom)} + const urlObj = new URL(document.location); + const hash = urlObj.hash; + if (hash) { + const [lat, lng, zoom] = hash.substring(1).split(","); + return { + centre: [parseFloat(lng), parseFloat(lat)], + zoom: parseFloat(zoom), + }; } - return {centre: undefined, zoom: undefined} + return { centre: undefined, zoom: undefined }; } async createMap() { @@ -62,7 +73,7 @@ export default class MapController { await getFreshApiToken(); - const viewFromUrl = this.getViewFromUrl() + const viewFromUrl = this.getViewFromUrl(); var map = new maplibregl.Map({ container: this.mapId, @@ -70,29 +81,27 @@ export default class MapController { maxZoom: 18, style: this.customStyleJson, maxBounds: [ - [ -15, 49 ], - [ 13, 57 ] + [-15, 49], + [13, 57], ], - center: viewFromUrl.centre || [ -1, 52.9 ], + center: viewFromUrl.centre || [-1, 52.9], zoom: viewFromUrl.zoom || 5.5, transformRequest: (url, resourceType) => { - if(url.indexOf('api.os.uk') > -1){ - if(! /[?&]key=/.test(url) ) url += '?key=null' - - const requestToMake = { - url: url + '&srs=3857', + if (url.startsWith(this.datasetVectorUrl)) { + // Check if the request URL is for your tile server + const newUrl = new URL(url); + if (this.useOAuth2) { + return { + url: newUrl.toString(), + headers: { Authorization: "Bearer " + getApiToken() }, + }; + } else { + newUrl.searchParams.append("key", this.apiKey); + return { url: newUrl.toString() }; } - - if(this.useOAuth2){ - const token = getApiToken(); - requestToMake.headers = { - 'Authorization': 'Bearer ' + token, - } - } - - return requestToMake; } - } + return { url }; + }, }); map.getCanvas().ariaLabel = `${this.mapId}`; @@ -100,206 +109,227 @@ export default class MapController { // once the maplibre map has loaded call the setup function var boundSetup = this.setup.bind(this); - this.map.on('load', boundSetup); - - }; + this.map.on("load", boundSetup); + } async setup() { - console.log('setup') - try{ + console.log("setup"); + try { await this.loadImages(this.images); - }catch(e){ - console.log('error loading images: ' + e) + } catch (e) { + console.log("error loading images: " + e); } - console.log('past load images') + console.log("past load images"); this.availableLayers = this.addVectorTileSources(this.vectorTileSources); this.geojsonLayers = this.addGeojsonSources(this.geojsons); - if(this.geojsonLayers.length == 1){ + if (this.geojsonLayers.length == 1) { this.flyTo(this.geojsons[0]); } - this.addControls() + this.addControls(); this.addClickHandlers(); this.overwriteWheelEventsForControls(); const handleMapMove = () => { - const center = this.map.getCenter() - const zoom = this.map.getZoom() - const urlObj = new URL(document.location) - const newURL = urlObj.origin + urlObj.pathname + urlObj.search + `#${center.lat},${center.lng},${zoom}z`; - window.history.replaceState({}, '', newURL); - } - this.obscureScotland() - this.obscureWales() - this.addNeighbours() - this.map.on('moveend',handleMapMove) - }; - - loadImages(imageSrc=[]) { - console.log('loading images' + imageSrc.length + ' images') + const center = this.map.getCenter(); + const zoom = this.map.getZoom(); + const urlObj = new URL(document.location); + const newURL = + urlObj.origin + + urlObj.pathname + + urlObj.search + + `#${center.lat},${center.lng},${zoom}z`; + window.history.replaceState({}, "", newURL); + }; + this.obscureScotland(); + this.obscureWales(); + this.addNeighbours(); + this.map.on("moveend", handleMapMove); + } + + loadImages(imageSrc = []) { + console.log("loading images" + imageSrc.length + " images"); return new Promise((resolve, reject) => { - const promiseArray = imageSrc.map(({src, name}) => { + const promiseArray = imageSrc.map(({ src, name }) => { return new Promise((resolve, reject) => { - this.map.loadImage( - src, - (error, image) => { - if (error){ - console.log('error adding image: ' + error) - reject(error); - } - console.log('added image') - this.map.addImage(name, image, {sdf: true}); - resolve(); + this.map.loadImage(src, (error, image) => { + if (error) { + console.log("error adding image: " + error); + reject(error); } - ); - }) - }); - Promise.all(promiseArray).then(() => { - console.log('resolved') - resolve(); - }).catch((error) => { - console.log('rejected') - reject(error); + console.log("added image"); + this.map.addImage(name, image, { sdf: true }); + resolve(); + }); + }); }); - }) + Promise.all(promiseArray) + .then(() => { + console.log("resolved"); + resolve(); + }) + .catch((error) => { + console.log("rejected"); + reject(error); + }); + }); } - addVectorTileSources(vectorTileSources = []) { + addVectorTileSources(vectorTileSources = []) { let availableLayers = {}; - // add vector tile sources to map - vectorTileSources.forEach(source => { + // add vector tile sources to map + vectorTileSources.forEach((source) => { let layers = this.addVectorTileSource(source); availableLayers[source.name] = layers; }); - return availableLayers; - } - - obscureWales(){ - this.obscure('Wales_simplified', '#FFFFFF', 0.6); + return availableLayers; } - obscureScotland(){ - this.obscure('Scotland_simplified'); + obscureWales() { + this.obscure("Wales_simplified", "#FFFFFF", 0.6); } - addNeighbours(){ - this.obscure('UK_neighbours', '#FFFFFF', 0.9); + obscureScotland() { + this.obscure("Scotland_simplified"); } + addNeighbours() { + this.obscure("UK_neighbours", "#FFFFFF", 0.9); + } - obscure(name, colour = '#FFFFFF', opacity = 0.8){ + obscure(name, colour = "#FFFFFF", opacity = 0.8) { this.map.addSource(name, { - type: 'geojson', + type: "geojson", data: `/static/javascripts/geojsons/${name}.json`, buffer: 0, - }) - const layerId = `${name}_Layer` + }); + const layerId = `${name}_Layer`; this.map.addLayer({ id: layerId, - type: 'fill', + type: "fill", source: name, layout: {}, paint: { - 'fill-color': colour, - 'fill-opacity': opacity, - } - }) - this.map.moveLayer(layerId,'OS/Names/National/Country') + "fill-color": colour, + "fill-opacity": opacity, + }, + }); + this.map.moveLayer(layerId, "OS/Names/National/Country"); } - addGeojsonSources(geojsons = []) { // add geojsons sources to map const addedLayers = []; - geojsons.forEach(geojson => { - if(geojson.data.type == 'Point') + geojsons.forEach((geojson) => { + if (geojson.data.type == "Point") addedLayers.push(this.addPoint(geojson, this.images[0])); - else if(['Polygon', 'MultiPolygon'].includes(geojson.data.type)) + else if (["Polygon", "MultiPolygon"].includes(geojson.data.type)) addedLayers.push(this.addPolygon(geojson)); - else - throw new Error('Unsupported geometry type'); + else throw new Error("Unsupported geometry type"); }); return addedLayers; } addControls() { - this.map.addControl(new maplibregl.ScaleControl({ - container: document.getElementById(this.mapId) - }), 'bottom-left'); - - if(this.FullscreenControl.enabled){ - this.map.addControl(new maplibregl.FullscreenControl({ - container: document.getElementById(this.mapId) - }), 'top-left'); - } - this.map.addControl(new TiltControl(), 'top-left'); - this.map.addControl(new maplibregl.NavigationControl({ - container: document.getElementById(this.mapId) - }), 'top-left'); + this.map.addControl( + new maplibregl.ScaleControl({ + container: document.getElementById(this.mapId), + }), + "bottom-left" + ); - this.map.addControl(new CopyrightControl(), 'bottom-right'); + if (this.FullscreenControl.enabled) { + this.map.addControl( + new maplibregl.FullscreenControl({ + container: document.getElementById(this.mapId), + }), + "top-left" + ); + } + this.map.addControl(new TiltControl(), "top-left"); + this.map.addControl( + new maplibregl.NavigationControl({ + container: document.getElementById(this.mapId), + }), + "top-left" + ); - if(this.LayerControlOptions.enabled){ - this.layerControlsComponent = new LayerControls(this, this.sourceName, this.layers, this.availableLayers, this.LayerControlOptions); - this.map.addControl(this.layerControlsComponent, 'top-right'); + this.map.addControl(new CopyrightControl(), "bottom-right"); + + if (this.LayerControlOptions.enabled) { + this.layerControlsComponent = new LayerControls( + this, + this.sourceName, + this.layers, + this.availableLayers, + this.LayerControlOptions + ); + this.map.addControl(this.layerControlsComponent, "top-right"); } } overwriteWheelEventsForControls() { - const mapEl = document.getElementById(this.mapId) - const mapControlsArray = mapEl.querySelectorAll('.maplibregl-control-container') - mapControlsArray.forEach((mapControls) => mapControls.addEventListener('wheel', preventScroll(['.dl-map__side-panel__content']), {passive: false})); + const mapEl = document.getElementById(this.mapId); + const mapControlsArray = mapEl.querySelectorAll( + ".maplibregl-control-container" + ); + mapControlsArray.forEach((mapControls) => + mapControls.addEventListener( + "wheel", + preventScroll([".dl-map__side-panel__content"]), + { passive: false } + ) + ); } addClickHandlers() { - if(this.layerControlsComponent){ - this.map.on('click', this.clickHandler.bind(this)); + if (this.layerControlsComponent) { + this.map.on("click", this.clickHandler.bind(this)); } } - - flyTo(geometry){ - if(geometry.data.type == 'Point'){ + flyTo(geometry) { + if (geometry.data.type == "Point") { this.map.flyTo({ center: geometry.data.coordinates, essential: true, animate: false, - zoom: 15 + zoom: 15, }); } else { var bbox = turf.extent(geometry.data); - this.map.fitBounds(bbox, {padding: 20, animate: false}); + this.map.fitBounds(bbox, { padding: 20, animate: false }); } } addLayer({ sourceName, layerType, - paintOptions={}, - layoutOptions={}, - sourceLayer='', - additionalOptions={} - }){ + paintOptions = {}, + layoutOptions = {}, + sourceLayer = "", + additionalOptions = {}, + }) { const layerName = `${sourceName}-${layerType}`; this.map.addLayer({ id: layerName, type: layerType, source: sourceName, - 'source-layer': sourceLayer, + "source-layer": sourceLayer, paint: paintOptions, layout: layoutOptions, - ...additionalOptions + ...additionalOptions, }); - if(['fill', 'fill-extrusion', 'circle'].includes(layerType)){ - this.map.on('mouseover', layerName, () => { - this.map.getCanvas().style.cursor = 'pointer' + if (["fill", "fill-extrusion", "circle"].includes(layerType)) { + this.map.on("mouseover", layerName, () => { + this.map.getCanvas().style.cursor = "pointer"; this.featuresHoveringOver++; - }) - this.map.on('mouseout', layerName, () => { + }); + this.map.on("mouseout", layerName, () => { this.featuresHoveringOver--; - if(this.featuresHoveringOver == 0) - this.map.getCanvas().style.cursor = '' - }) + if (this.featuresHoveringOver == 0) + this.map.getCanvas().style.cursor = ""; + }); } return layerName; @@ -307,151 +337,158 @@ export default class MapController { addPolygon(geometry) { this.map.addSource(geometry.name, { - 'type': 'geojson', - 'data': { - 'type': 'Feature', - 'geometry': geometry.data, - 'properties': { - 'entity': geometry.entity, - 'name': geometry.name, - } + type: "geojson", + data: { + type: "Feature", + geometry: geometry.data, + properties: { + entity: geometry.entity, + name: geometry.name, + }, }, }); - let colour = 'blue'; - if(this.paint_options) - colour = this.paint_options.colour; + let colour = "blue"; + if (this.paint_options) colour = this.paint_options.colour; let layer = this.addLayer({ sourceName: geometry.name, - layerType: 'fill-extrusion', + layerType: "fill-extrusion", paintOptions: { - 'fill-extrusion-color': colour, - 'fill-extrusion-opacity': 0.5, - 'fill-extrusion-height': 1, - 'fill-extrusion-base': 0, + "fill-extrusion-color": colour, + "fill-extrusion-opacity": 0.5, + "fill-extrusion-height": 1, + "fill-extrusion-base": 0, }, }); - this.moveLayerBehindBuildings(layer) + this.moveLayerBehindBuildings(layer); return layer; } - moveLayerBehindBuildings(layer, buildingsLayer = 'OS/TopographicArea_1/Building/1_3D') { - try{ + moveLayerBehindBuildings( + layer, + buildingsLayer = "OS/TopographicArea_1/Building/1_3D" + ) { + try { this.map.moveLayer(layer, buildingsLayer); } catch (e) { console.error(`Could not move layer behind ${buildingsLayer}: `, e); } } - addPoint(geometry, image=undefined){ + addPoint(geometry, image = undefined) { this.map.addSource(geometry.name, { - 'type': 'geojson', - 'data': { - 'type': 'Feature', - 'geometry': geometry.data, - 'properties': { - 'entity': geometry.entity, - 'name': geometry.name, - } - } + type: "geojson", + data: { + type: "Feature", + geometry: geometry.data, + properties: { + entity: geometry.entity, + name: geometry.name, + }, + }, }); - let iconColor = 'blue'; - if(this.paint_options) - iconColor = this.paint_options.colour; + let iconColor = "blue"; + if (this.paint_options) iconColor = this.paint_options.colour; - let layerName + let layerName; // if an image is provided use that otherwise use a circle - if(image){ - if(!this.map.hasImage(image.name)){ - throw new Error('Image not loaded, imageName: ' + image.name + ' not found'); + if (image) { + if (!this.map.hasImage(image.name)) { + throw new Error( + "Image not loaded, imageName: " + image.name + " not found" + ); } - layerName = this.addLayer( - { - sourceName: geometry.name, - layerType: 'symbol', - paintOptions: { - 'icon-color': iconColor, - 'icon-opacity': 1, - }, - layoutOptions: { - 'icon-image': image.name, - 'icon-size': 256 / image.size * 0.15, - 'icon-anchor': 'bottom', - // get the year from the source's "year" property - 'text-field': ['get', 'year'], - 'text-font': [ - 'Open Sans Semibold', - 'Arial Unicode MS Bold' - ], - 'text-offset': [0, 1.25], - 'text-anchor': 'top' - }, - }) - }else{ layerName = this.addLayer({ sourceName: geometry.name, - layerType: 'circle', + layerType: "symbol", paintOptions: { - 'circle-color': iconColor, - "circle-radius": defaultPaintOptions['circle-radius'], - } - }) + "icon-color": iconColor, + "icon-opacity": 1, + }, + layoutOptions: { + "icon-image": image.name, + "icon-size": (256 / image.size) * 0.15, + "icon-anchor": "bottom", + // get the year from the source's "year" property + "text-field": ["get", "year"], + "text-font": ["Open Sans Semibold", "Arial Unicode MS Bold"], + "text-offset": [0, 1.25], + "text-anchor": "top", + }, + }); + } else { + layerName = this.addLayer({ + sourceName: geometry.name, + layerType: "circle", + paintOptions: { + "circle-color": iconColor, + "circle-radius": defaultPaintOptions["circle-radius"], + }, + }); } return layerName; } addVectorTileSource(source) { - // add source - this.map.addSource(`${source.name}-source`, { - type: 'vector', - tiles: [source.vectorSource], - minzoom: this.minMapZoom, - maxzoom: this.maxMapZoom - }); - - // add layer - let layers; - if (source.dataType === 'point') { + // add source + this.map.addSource(`${source.name}-source`, { + type: "vector", + tiles: [source.vectorSource], + minzoom: this.minMapZoom, + maxzoom: this.maxMapZoom, + }); + + // add layer + let layers; + if (source.dataType === "point") { let layerName = this.addLayer({ sourceName: `${source.name}-source`, - layerType: 'circle', + layerType: "circle", paintOptions: { - 'circle-color': source.styleProps.colour || defaultPaintOptions['fill-color'], - 'circle-opacity': source.styleProps.opacity || defaultPaintOptions['fill-opacity'], - 'circle-stroke-color': source.styleProps.colour || defaultPaintOptions['fill-color'], - "circle-radius": defaultPaintOptions['circle-radius'] + "circle-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "circle-opacity": + source.styleProps.opacity || defaultPaintOptions["fill-opacity"], + "circle-stroke-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "circle-radius": defaultPaintOptions["circle-radius"], }, sourceLayer: `${source.name}`, }); - layers = [layerName]; - } else { - // create fill layer + layers = [layerName]; + } else { + // create fill layer let fillLayerName = this.addLayer({ sourceName: `${source.name}-source`, - layerType: 'fill-extrusion', + layerType: "fill-extrusion", paintOptions: { - 'fill-extrusion-color': source.styleProps.colour || defaultPaintOptions['fill-color'], - 'fill-extrusion-height': 1, - 'fill-extrusion-base': 0, - 'fill-extrusion-opacity': parseFloat(source.styleProps.opacity) || defaultPaintOptions['fill-opacity'] + "fill-extrusion-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "fill-extrusion-height": 1, + "fill-extrusion-base": 0, + "fill-extrusion-opacity": + parseFloat(source.styleProps.opacity) || + defaultPaintOptions["fill-opacity"], }, sourceLayer: `${source.name}`, }); - this.moveLayerBehindBuildings(fillLayerName) + this.moveLayerBehindBuildings(fillLayerName); - // create line layer + // create line layer let lineLayerName = this.addLayer({ sourceName: `${source.name}-source`, - layerType: 'line', + layerType: "line", paintOptions: { - 'line-color': source.styleProps.colour || defaultPaintOptions['fill-color'], - 'line-width': source.styleProps.weight || defaultPaintOptions['weight'] + "line-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "line-width": + source.styleProps.weight || defaultPaintOptions["weight"], }, sourceLayer: `${source.name}`, }); @@ -459,43 +496,55 @@ export default class MapController { // create point layer for geometries let pointLayerName = this.addLayer({ sourceName: `${source.name}-source`, - layerType: 'circle', + layerType: "circle", paintOptions: { - 'circle-color': source.styleProps.colour || defaultPaintOptions['fill-color'], - 'circle-opacity': source.styleProps.opacity || defaultPaintOptions['fill-opacity'], - 'circle-stroke-color': source.styleProps.colour || defaultPaintOptions['fill-color'], - "circle-radius": defaultPaintOptions['circle-radius'] + "circle-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "circle-opacity": + source.styleProps.opacity || defaultPaintOptions["fill-opacity"], + "circle-stroke-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "circle-radius": defaultPaintOptions["circle-radius"], }, sourceLayer: `${source.name}`, - additionalOptions:{ - filter:[ "==", ["geometry-type"], "Point"] + additionalOptions: { + filter: ["==", ["geometry-type"], "Point"], }, }); - layers = [fillLayerName, lineLayerName, pointLayerName]; - } - return layers; + layers = [fillLayerName, lineLayerName, pointLayerName]; + } + return layers; } clickHandler(e) { var map = this.map; - var bbox = [[e.point.x - 5, e.point.y - 5], [e.point.x + 5, e.point.y + 5]]; + var bbox = [ + [e.point.x - 5, e.point.y - 5], + [e.point.x + 5, e.point.y + 5], + ]; - let clickableLayers = this.layerControlsComponent.getClickableLayers() || []; + let clickableLayers = + this.layerControlsComponent.getClickableLayers() || []; var features = map.queryRenderedFeatures(bbox, { - layers: clickableLayers + layers: clickableLayers, }); var coordinates = e.lngLat; if (features.length) { // no need to show popup if not clicking on feature - var popupDomElement = this.createFeaturesPopup(this.removeDuplicates(features)); + var popupDomElement = this.createFeaturesPopup( + this.removeDuplicates(features) + ); var popup = new maplibregl.Popup({ - maxWidth: this.popupWidth - }).setLngLat(coordinates).setDOMContent(popupDomElement).addTo(map); - popup.getElement().onwheel = preventScroll(['.app-popup-list']); + maxWidth: this.popupWidth, + }) + .setLngLat(coordinates) + .setDOMContent(popupDomElement) + .addTo(map); + popup.getElement().onwheel = preventScroll([".app-popup-list"]); } - }; + } // map.queryRenderedFeatures() can return duplicate features so we need to remove them removeDuplicates(features) { @@ -509,53 +558,64 @@ export default class MapController { return false; }); - - }; + } createFeaturesPopup(features) { - const wrapper = document.createElement('div'); - wrapper.classList.add('app-popup'); + const wrapper = document.createElement("div"); + wrapper.classList.add("app-popup"); - const featureOrFeatures = features.length > 1 ? 'features' : 'feature'; - const heading = document.createElement('h3'); - heading.classList.add('app-popup-heading'); + const featureOrFeatures = features.length > 1 ? "features" : "feature"; + const heading = document.createElement("h3"); + heading.classList.add("app-popup-heading"); heading.textContent = `${features.length} ${featureOrFeatures} selected`; wrapper.appendChild(heading); if (features.length > this.popupMaxListLength) { - const tooMany = document.createElement('p'); - tooMany.classList.add('govuk-body-s'); + const tooMany = document.createElement("p"); + tooMany.classList.add("govuk-body-s"); tooMany.textContent = `You clicked on ${features.length} features.`; - const tooMany2 = document.createElement('p'); - tooMany2.classList.add('govuk-body-s'); - tooMany2.textContent = 'Zoom in or turn off layers to narrow down your choice.'; + const tooMany2 = document.createElement("p"); + tooMany2.classList.add("govuk-body-s"); + tooMany2.textContent = + "Zoom in or turn off layers to narrow down your choice."; wrapper.appendChild(tooMany); wrapper.appendChild(tooMany2); return wrapper; } - const list = document.createElement('ul'); - list.classList.add('app-popup-list'); + const list = document.createElement("ul"); + list.classList.add("app-popup-list"); features.forEach((feature) => { - const featureType = capitalizeFirstLetter(feature.sourceLayer || feature.source).replaceAll('-', ' '); + const featureType = capitalizeFirstLetter( + feature.sourceLayer || feature.source + ).replaceAll("-", " "); const fillColour = this.getFillColour(feature); - const featureName = feature.properties.name || feature.properties.reference || 'Not Named'; - const item = document.createElement('li'); - item.classList.add('app-popup-item'); + const featureName = + feature.properties.name || feature.properties.reference || "Not Named"; + const item = document.createElement("li"); + item.classList.add("app-popup-item"); item.style.borderLeft = `5px solid ${fillColour}`; - const secondaryText = document.createElement('p'); - secondaryText.classList.add('app-u-secondary-text', 'govuk-!-margin-bottom-0', 'govuk-!-margin-top-0'); + const secondaryText = document.createElement("p"); + secondaryText.classList.add( + "app-u-secondary-text", + "govuk-!-margin-bottom-0", + "govuk-!-margin-top-0" + ); secondaryText.textContent = featureType; item.appendChild(secondaryText); - const link = document.createElement('a'); - link.classList.add('govuk-link'); + const link = document.createElement("a"); + link.classList.add("govuk-link"); link.href = `/entity/${feature.properties.entity}`; link.textContent = featureName; - const smallText = document.createElement('p'); - smallText.classList.add('dl-small-text', 'govuk-!-margin-top-0', 'govuk-!-margin-bottom-0'); + const smallText = document.createElement("p"); + smallText.classList.add( + "dl-small-text", + "govuk-!-margin-top-0", + "govuk-!-margin-bottom-0" + ); smallText.appendChild(link); item.appendChild(smallText); @@ -564,27 +624,28 @@ export default class MapController { wrapper.appendChild(list); return wrapper; - }; + } getFillColour(feature) { - if(feature.layer.type === 'symbol') - return this.map.getLayer(feature.layer.id).getPaintProperty('icon-color'); - else if(feature.layer.type === 'fill') - return this.map.getLayer(feature.layer.id).getPaintProperty('fill-color'); - else if(feature.layer.type === 'fill-extrusion') - return this.map.getLayer(feature.layer.id).getPaintProperty('fill-extrusion-color'); - else if(feature.layer.type === 'circle') - return this.map.getLayer(feature.layer.id).getPaintProperty('circle-color'); + if (feature.layer.type === "symbol") + return this.map.getLayer(feature.layer.id).getPaintProperty("icon-color"); + else if (feature.layer.type === "fill") + return this.map.getLayer(feature.layer.id).getPaintProperty("fill-color"); + else if (feature.layer.type === "fill-extrusion") + return this.map + .getLayer(feature.layer.id) + .getPaintProperty("fill-extrusion-color"); + else if (feature.layer.type === "circle") + return this.map + .getLayer(feature.layer.id) + .getPaintProperty("circle-color"); else - throw new Error("could not get fill colour for feature of type " + feature.layer.type); - }; + throw new Error( + "could not get fill colour for feature of type " + feature.layer.type + ); + } setLayerVisibility(layerName, visibility) { - this.map.setLayoutProperty( - layerName, - 'visibility', - visibility - ); - }; - + this.map.setLayoutProperty(layerName, "visibility", visibility); + } } From 1172ce75b4ebc650fa021bbab4ac77575f66b0b2 Mon Sep 17 00:00:00 2001 From: James Bannister Date: Thu, 2 May 2024 11:59:37 +0100 Subject: [PATCH 03/16] Test update --- .../javascript/MapController.test.js | 1052 +++++++++-------- 1 file changed, 558 insertions(+), 494 deletions(-) diff --git a/tests/integration/javascript/MapController.test.js b/tests/integration/javascript/MapController.test.js index b2661c4e..00a9a7e5 100644 --- a/tests/integration/javascript/MapController.test.js +++ b/tests/integration/javascript/MapController.test.js @@ -1,25 +1,25 @@ // integration tests for the map controller -import {describe, expect, test, vi, beforeEach} from 'vitest' -import MapController from '../../../assets/javascripts/MapController' -import TiltControl from '../../../assets/javascripts/TiltControl'; +import { describe, expect, test, vi, beforeEach } from "vitest"; +import MapController from "../../../assets/javascripts/MapController"; +import TiltControl from "../../../assets/javascripts/TiltControl"; import { - getDomElementMock, - getMapMock, - getUrlDeleteMock, - getUrlAppendMock, - getPopupMock, - stubGlobalMapLibre, - stubGlobalWindow, - stubGlobalUrl, - stubGlobalDocument, - stubGlobalFetch, - waitForMapCreation -} from '../../utils/mockUtils'; -import CopyrightControl from '../../../assets/javascripts/CopyrightControl'; + getDomElementMock, + getMapMock, + getUrlDeleteMock, + getUrlAppendMock, + getPopupMock, + stubGlobalMapLibre, + stubGlobalWindow, + stubGlobalUrl, + stubGlobalDocument, + stubGlobalFetch, + waitForMapCreation, +} from "../../utils/mockUtils"; +import CopyrightControl from "../../../assets/javascripts/CopyrightControl"; stubGlobalMapLibre(); -stubGlobalWindow('http://localhost', '') +stubGlobalWindow("http://localhost", ""); const [urlDeleteMock, urlAppendMock] = stubGlobalUrl(); stubGlobalDocument(); stubGlobalFetch(); @@ -29,484 +29,548 @@ let mapMock = getMapMock(); let popupMock = getPopupMock(); beforeEach(() => { - vi.clearAllMocks() -}) - -describe('Map Controller', () => { - describe('Constructor', () => { - test('Works as expected, applying default params', async () => { - const mapController = new MapController({ - images: [ - { - src: '/static/images/location-pointer-sdf.png', - name: 'custom-marker', - } - ] - }) - await waitForMapCreation(mapController) - expect(mapController.map.events.load).toBeDefined() - - await mapController.map.events.load() // initiate the load event - - expect(mapController).toBeDefined() - expect(mapController.map).toBeDefined() - - expect(mapController.mapId).toEqual('mapid'); - expect(mapController.mapContainerSelector).toEqual('.dl-map__wrapper'); - expect(mapController.vectorTileSources).toEqual([]); - expect(mapController.datasetVectorUrl).toEqual(null); - expect(mapController.datasets).toEqual(null); - expect(mapController.minMapZoom).toEqual(5); - expect(mapController.maxMapZoom).toEqual(15); - expect(mapController.baseURL).toEqual('https://digital-land.github.io'); - expect(mapController.baseTileStyleFilePath).toEqual('/static/javascripts/base-tile.json'); - expect(mapController.popupWidth).toEqual('260px'); - expect(mapController.popupMaxListLength).toEqual(10); - expect(mapController.LayerControlOptions).toEqual({enabled: false}); - expect(mapController.ZoomControlsOptions).toEqual({enabled: false}); - expect(mapController.FullscreenControl).toEqual({enabled: false}); - expect(mapController.geojsons).toEqual([]); - expect(mapController.images).toEqual([{src: '/static/images/location-pointer-sdf.png', name: 'custom-marker'}]); - expect(mapController.paint_options).toEqual(null); - - expect(mapController.map.loadImage).toHaveBeenCalledOnce(); - expect(mapController.map.addImage).toHaveBeenCalledOnce(); - expect(mapController.map.loadImage).toHaveBeenCalledWith('/static/images/location-pointer-sdf.png', expect.any(Function)); - expect(mapController.map.addImage).toHaveBeenCalledWith('custom-marker', 'the Image', {sdf: true}); - - expect(mapController.map.addControl).toHaveBeenCalledTimes(4); - expect(mapController.map.addControl).toHaveBeenCalledWith(new maplibregl.ScaleControl, 'top-left'); - expect(mapController.map.addControl).toHaveBeenCalledWith(new maplibregl.NavigationControl, 'top-left'); - expect(mapController.map.addControl).toHaveBeenCalledWith(new TiltControl, 'top-left'); - expect(mapController.map.addControl).toHaveBeenCalledWith(new CopyrightControl, 'bottom-right'); - }) - - test('Works as expected, enabling full screen', async () => { - const mapController = new MapController({ - FullscreenControl: { - enabled: true, - } - }) - await waitForMapCreation(mapController) - expect(mapController.map.events.load).toBeDefined() - - await mapController.map.events.load() // initiate the load event - - expect(mapController).toBeDefined() - expect(mapController.map).toBeDefined() - - expect(mapController.mapId).toEqual('mapid'); - expect(mapController.mapContainerSelector).toEqual('.dl-map__wrapper'); - expect(mapController.vectorTileSources).toEqual([]); - expect(mapController.datasetVectorUrl).toEqual(null); - expect(mapController.datasets).toEqual(null); - expect(mapController.minMapZoom).toEqual(5); - expect(mapController.maxMapZoom).toEqual(15); - expect(mapController.baseURL).toEqual('https://digital-land.github.io'); - expect(mapController.baseTileStyleFilePath).toEqual('/static/javascripts/base-tile.json'); - expect(mapController.popupWidth).toEqual('260px'); - expect(mapController.popupMaxListLength).toEqual(10); - expect(mapController.LayerControlOptions).toEqual({enabled: false}); - expect(mapController.ZoomControlsOptions).toEqual({enabled: false}); - expect(mapController.FullscreenControl).toEqual({enabled: true}); - expect(mapController.geojsons).toEqual([]); - expect(mapController.images).toEqual([{src: '/static/images/location-pointer-sdf-256.png', name: 'custom-marker-256', size: 256}]); - expect(mapController.paint_options).toEqual(null); - - expect(mapController.map.loadImage).toHaveBeenCalledOnce(); - expect(mapController.map.addImage).toHaveBeenCalledOnce(); - expect(mapController.map.loadImage).toHaveBeenCalledWith('/static/images/location-pointer-sdf-256.png', expect.any(Function)); - expect(mapController.map.addImage).toHaveBeenCalledWith('custom-marker-256', 'the Image', {sdf: true}); - - expect(mapController.map.addControl).toHaveBeenCalledTimes(5); - expect(mapController.map.addControl).toHaveBeenCalledWith(new maplibregl.ScaleControl, 'top-left'); - expect(mapController.map.addControl).toHaveBeenCalledWith(new maplibregl.NavigationControl, 'top-left'); - expect(mapController.map.addControl).toHaveBeenCalledWith(new maplibregl.FullscreenControl, 'bottom-left'); - expect(mapController.map.addControl).toHaveBeenCalledWith(new TiltControl, 'top-left'); - expect(mapController.map.addControl).toHaveBeenCalledWith(new CopyrightControl, 'bottom-right'); - }) - - test('Works with one geojson feature of type point', async () => { - const params = { - geojsons: [ - { - name: 'testName', - data: { - type: 'Point', - }, - entity: 'testEntity', - } - ], - paint_options: { - colour: '#0000ff', - } - } - - const mapController = new MapController(params) - await waitForMapCreation(mapController) - await mapController.map.events.load() // initiate the load event - - expect(mapController.map.addSource).toHaveBeenCalledTimes(4); - expect(mapController.map.addLayer).toHaveBeenCalledTimes(4); - expect(mapController.map.addSource).toHaveBeenCalledWith(params.geojsons[0].name, { - 'type': 'geojson', - 'data': { - 'type': 'Feature', - 'geometry': params.geojsons[0].data, - 'properties': { - 'entity': params.geojsons[0].entity, - 'name': params.geojsons[0].name, - } - } - }); - const layerName = `${params.geojsons[0].name}-symbol`; - expect(mapController.map.addLayer).toHaveBeenCalledWith({ - id: layerName, - type: 'symbol', - source: params.geojsons[0].name, - 'source-layer': '', - paint: { - 'icon-color': params.paint_options.colour, - 'icon-opacity': 1, - }, - layout: { - 'icon-image': 'custom-marker-256', - 'icon-size': 0.15, - 'icon-anchor': 'bottom', - // get the year from the source's "year" property - 'text-field': ['get', 'year'], - 'text-font': [ - 'Open Sans Semibold', - 'Arial Unicode MS Bold' - ], - 'text-offset': [0, 1.25], - 'text-anchor': 'top' - } - }) - }) - - test('Works with many geojson features of type point', async () => { - const params = { - geojsons: [ - { - name: 'testName', - data: { - type: 'Point', - } - }, - { - name: 'testName1', - data: { - type: 'Point', - } - }, - { - name: 'testName2', - data: { - type: 'Point', - } - } - ] - } - - const mapController = new MapController(params) - await waitForMapCreation(mapController) - await mapController.map.events.load() // initiate the load event - - expect(mapController.map.addSource).toHaveBeenCalledTimes(6); - expect(mapController.map.addLayer).toHaveBeenCalledTimes(6); - - params.geojsons.forEach((geojson, index) => { - expect(mapController.map.addSource).toHaveBeenCalledWith(params.geojsons[index].name, { - 'type': 'geojson', - 'data': { - 'type': 'Feature', - 'geometry': params.geojsons[index].data, - 'properties': { - 'entity': params.geojsons[index].entity, - 'name': params.geojsons[index].name, - } - } - }); - const layerName = `${params.geojsons[index].name}-symbol`; - expect(mapController.map.addLayer).toHaveBeenCalledWith({ - id: layerName, - type: 'symbol', - source: params.geojsons[index].name, - 'source-layer': '', - paint: { - 'icon-color': 'blue', - 'icon-opacity': 1, - }, - layout: { - 'icon-image': 'custom-marker-256', - 'icon-size': 0.15, - 'icon-anchor': 'bottom', - // get the year from the source's "year" property - 'text-field': ['get', 'year'], - 'text-font': [ - 'Open Sans Semibold', - 'Arial Unicode MS Bold' - ], - 'text-offset': [0, 1.25], - 'text-anchor': 'top' - } - }) - }) - }) - - test('Works with many geojson features of type polygon/MultiPolygon', async () => { - const params = { - geojsons: [ - { - name: 'testName', - data: { - type: 'Polygon', - } - }, - { - name: 'testName1', - data: { - type: 'Polygon', - } - }, - { - name: 'testName2', - data: { - type: 'MultiPolygon', - } - } - ] - } - - const mapController = new MapController(params) - await waitForMapCreation(mapController) - await mapController.map.events.load() // initiate the load event - - expect(mapController.map.addSource).toHaveBeenCalledTimes(6); - expect(mapController.map.addLayer).toHaveBeenCalledTimes(6); - - params.geojsons.forEach((geojson, index) => { - expect(mapController.map.addSource).toHaveBeenCalledWith(params.geojsons[index].name, { - 'type': 'geojson', - 'data': { - 'type': 'Feature', - 'geometry': params.geojsons[index].data, - 'properties': { - 'entity': params.geojsons[index].entity, - 'name': params.geojsons[index].name, - } - } - }); - const layerName = `${params.geojsons[index].name}-fill-extrusion`; - expect(mapController.map.addLayer).toHaveBeenCalledWith({ - id: layerName, - type: 'fill-extrusion', - source: params.geojsons[index].name, - 'source-layer': '', - paint: { - 'fill-extrusion-color': 'blue', - 'fill-extrusion-opacity': 0.5, - 'fill-extrusion-height': 1, - 'fill-extrusion-base': 0, - }, - layout: {} - }) - }) - }) - - test('Works with many geojson features of type polygon with layer controls enabled', async () => { - const params = { - geojsons: [ - { - name: 'testName', - data: { - type: 'Polygon', - } - }, - { - name: 'testName1', - data: { - type: 'Polygon', - } - }, - { - name: 'testName2', - data: { - type: 'MultiPolygon', - } - } - ], - LayerControlOptions: { - enabled: true, - }, - } - - const mapController = new MapController(params) - await waitForMapCreation(mapController) - await mapController.map.events.load() // initiate the load event - - expect(mapController.layerControlsComponent).toBeDefined(); - }) - - test('Works with a point vectorSource layer', async () => { - const minMapZoom = 10; - const maxMapZoom = 20; - const params = { - vectorTileSources: [ - { - name: 'testName', - vectorSource: 'testUrl', - dataType: 'point', - styleProps: { - colour: '#0000ff', - opacity: 0.5, - } - } - ], - LayerControlOptions: { - enabled: true, - }, - minMapZoom: minMapZoom, - maxMapZoom: maxMapZoom, - } - - const mapController = new MapController(params) - await waitForMapCreation(mapController) - await mapController.map.events.load() // initiate the load event - - expect(mapController.map.addSource).toHaveBeenCalledTimes(4); - expect(mapController.map.addLayer).toHaveBeenCalledTimes(4); - expect(mapController.map.addSource).toHaveBeenCalledWith(params.vectorTileSources[0].name + '-source', { - type: 'vector', - tiles: [params.vectorTileSources[0].vectorSource], - minzoom: minMapZoom, - maxzoom: maxMapZoom - }); - expect(mapController.map.addLayer).toHaveBeenCalledWith({ - id: `${params.vectorTileSources[0].name}-source-circle`, - type: 'circle', - source: `${params.vectorTileSources[0].name}-source`, - 'source-layer': `${params.vectorTileSources[0].name}`, - paint: { - 'circle-color': params.vectorTileSources[0].styleProps.colour, - 'circle-opacity': params.vectorTileSources[0].styleProps.opacity, - 'circle-stroke-color': params.vectorTileSources[0].styleProps.colour, - 'circle-radius': [ - "interpolate", ["linear"], ["zoom"], - 6, 1, - 14, 8, - ], - }, - layout: {} - }); - expect(mapController.availableLayers).toEqual({ - [params.vectorTileSources[0].name]: [`${params.vectorTileSources[0].name}-source-circle`] - }) - }) - - test('Works with a polygon vectorSource layer', async () => { - const minMapZoom = 10; - const maxMapZoom = 20; - const params = { - vectorTileSources: [ - { - name: 'testName', - vectorSource: 'testUrl', - dataType: 'polygon', - styleProps: { - colour: '#0000ff', - opacity: 0.5, - }, - }, - ], - minMapZoom, - maxMapZoom, - LayerControlOptions: { - enabled: true, - } - } - - const mapController = new MapController(params) - await waitForMapCreation(mapController) - - await mapController.map.events.load() // initiate the load event - - expect(mapController.map.addSource).toHaveBeenCalledTimes(4); - expect(mapController.map.addLayer).toHaveBeenCalledTimes(6); - expect(mapController.map.addSource).toHaveBeenCalledWith(params.vectorTileSources[0].name + '-source', { - type: 'vector', - tiles: [params.vectorTileSources[0].vectorSource], - minzoom: minMapZoom, - maxzoom: maxMapZoom - }); - expect(mapController.map.addLayer).toHaveBeenCalledWith({ - id: `${params.vectorTileSources[0].name}-source-fill-extrusion`, - type: 'fill-extrusion', - source: `${params.vectorTileSources[0].name}-source`, - 'source-layer': `${params.vectorTileSources[0].name}`, - paint: { - 'fill-extrusion-color': params.vectorTileSources[0].styleProps.colour, - 'fill-extrusion-opacity': params.vectorTileSources[0].styleProps.opacity, - 'fill-extrusion-height': 1, - 'fill-extrusion-base': 0, - }, - layout: {} - }); - expect(mapController.map.addLayer).toHaveBeenCalledWith({ - id: `${params.vectorTileSources[0].name}-source-line`, - type: 'line', - source: `${params.vectorTileSources[0].name}-source`, - 'source-layer': `${params.vectorTileSources[0].name}`, - paint: { - 'line-color': params.vectorTileSources[0].styleProps.colour, - 'line-width': 1, - }, - layout: {} - }); - expect(mapController.availableLayers).toEqual({ - [params.vectorTileSources[0].name]: [ - `${params.vectorTileSources[0].name}-source-fill-extrusion`, - `${params.vectorTileSources[0].name}-source-line`, - `${params.vectorTileSources[0].name}-source-circle`, - ] - }) - }) - }) - - test('clickHandler works as expected', async () => { - const mapController = new MapController({ - LayerControlOptions: { - enabled: true, + vi.clearAllMocks(); +}); + +describe("Map Controller", () => { + describe("Constructor", () => { + test("Works as expected, applying default params", async () => { + const mapController = new MapController({ + images: [ + { + src: "/static/images/location-pointer-sdf.png", + name: "custom-marker", + }, + ], + }); + await waitForMapCreation(mapController); + expect(mapController.map.events.load).toBeDefined(); + + await mapController.map.events.load(); // initiate the load event + + expect(mapController).toBeDefined(); + expect(mapController.map).toBeDefined(); + + expect(mapController.mapId).toEqual("mapid"); + expect(mapController.mapContainerSelector).toEqual(".dl-map__wrapper"); + expect(mapController.vectorTileSources).toEqual([]); + expect(mapController.datasetVectorUrl).toEqual("http://"); + expect(mapController.datasets).toEqual(null); + expect(mapController.minMapZoom).toEqual(5); + expect(mapController.maxMapZoom).toEqual(15); + expect(mapController.baseURL).toEqual("https://digital-land.github.io"); + expect(mapController.baseTileStyleFilePath).toEqual( + "/static/javascripts/base-tile.json" + ); + expect(mapController.popupWidth).toEqual("260px"); + expect(mapController.popupMaxListLength).toEqual(10); + expect(mapController.LayerControlOptions).toEqual({ enabled: false }); + expect(mapController.ZoomControlsOptions).toEqual({ enabled: false }); + expect(mapController.FullscreenControl).toEqual({ enabled: false }); + expect(mapController.geojsons).toEqual([]); + expect(mapController.images).toEqual([ + { + src: "/static/images/location-pointer-sdf.png", + name: "custom-marker", + }, + ]); + expect(mapController.paint_options).toEqual(null); + + expect(mapController.map.loadImage).toHaveBeenCalledOnce(); + expect(mapController.map.addImage).toHaveBeenCalledOnce(); + expect(mapController.map.loadImage).toHaveBeenCalledWith( + "/static/images/location-pointer-sdf.png", + expect.any(Function) + ); + expect(mapController.map.addImage).toHaveBeenCalledWith( + "custom-marker", + "the Image", + { sdf: true } + ); + + expect(mapController.map.addControl).toHaveBeenCalledTimes(4); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new maplibregl.ScaleControl(), + "top-left" + ); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new maplibregl.NavigationControl(), + "top-left" + ); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new TiltControl(), + "top-left" + ); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new CopyrightControl(), + "bottom-right" + ); + }); + + test("Works as expected, enabling full screen", async () => { + const mapController = new MapController({ + FullscreenControl: { + enabled: true, + }, + }); + await waitForMapCreation(mapController); + expect(mapController.map.events.load).toBeDefined(); + + await mapController.map.events.load(); // initiate the load event + + expect(mapController).toBeDefined(); + expect(mapController.map).toBeDefined(); + + expect(mapController.mapId).toEqual("mapid"); + expect(mapController.mapContainerSelector).toEqual(".dl-map__wrapper"); + expect(mapController.vectorTileSources).toEqual([]); + expect(mapController.datasetVectorUrl).toEqual("http://"); + expect(mapController.datasets).toEqual(null); + expect(mapController.minMapZoom).toEqual(5); + expect(mapController.maxMapZoom).toEqual(15); + expect(mapController.baseURL).toEqual("https://digital-land.github.io"); + expect(mapController.baseTileStyleFilePath).toEqual( + "/static/javascripts/base-tile.json" + ); + expect(mapController.popupWidth).toEqual("260px"); + expect(mapController.popupMaxListLength).toEqual(10); + expect(mapController.LayerControlOptions).toEqual({ enabled: false }); + expect(mapController.ZoomControlsOptions).toEqual({ enabled: false }); + expect(mapController.FullscreenControl).toEqual({ enabled: true }); + expect(mapController.geojsons).toEqual([]); + expect(mapController.images).toEqual([ + { + src: "/static/images/location-pointer-sdf-256.png", + name: "custom-marker-256", + size: 256, + }, + ]); + expect(mapController.paint_options).toEqual(null); + + expect(mapController.map.loadImage).toHaveBeenCalledOnce(); + expect(mapController.map.addImage).toHaveBeenCalledOnce(); + expect(mapController.map.loadImage).toHaveBeenCalledWith( + "/static/images/location-pointer-sdf-256.png", + expect.any(Function) + ); + expect(mapController.map.addImage).toHaveBeenCalledWith( + "custom-marker-256", + "the Image", + { sdf: true } + ); + + expect(mapController.map.addControl).toHaveBeenCalledTimes(5); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new maplibregl.ScaleControl(), + "top-left" + ); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new maplibregl.NavigationControl(), + "top-left" + ); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new maplibregl.FullscreenControl(), + "bottom-left" + ); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new TiltControl(), + "top-left" + ); + expect(mapController.map.addControl).toHaveBeenCalledWith( + new CopyrightControl(), + "bottom-right" + ); + }); + + test("Works with one geojson feature of type point", async () => { + const params = { + geojsons: [ + { + name: "testName", + data: { + type: "Point", }, - }) - - await waitForMapCreation(mapController) - - await new Promise(resolve => setTimeout(resolve, 1000)) // wait for the map to load - - await mapController.map.events.load() // initiate the load event - - const mockClickEvent = { - point: { - x: 100, - y: 100, + entity: "testEntity", + }, + ], + paint_options: { + colour: "#0000ff", + }, + }; + + const mapController = new MapController(params); + await waitForMapCreation(mapController); + await mapController.map.events.load(); // initiate the load event + + expect(mapController.map.addSource).toHaveBeenCalledTimes(4); + expect(mapController.map.addLayer).toHaveBeenCalledTimes(4); + expect(mapController.map.addSource).toHaveBeenCalledWith( + params.geojsons[0].name, + { + type: "geojson", + data: { + type: "Feature", + geometry: params.geojsons[0].data, + properties: { + entity: params.geojsons[0].entity, + name: params.geojsons[0].name, + }, + }, + } + ); + const layerName = `${params.geojsons[0].name}-symbol`; + expect(mapController.map.addLayer).toHaveBeenCalledWith({ + id: layerName, + type: "symbol", + source: params.geojsons[0].name, + "source-layer": "", + paint: { + "icon-color": params.paint_options.colour, + "icon-opacity": 1, + }, + layout: { + "icon-image": "custom-marker-256", + "icon-size": 0.15, + "icon-anchor": "bottom", + // get the year from the source's "year" property + "text-field": ["get", "year"], + "text-font": ["Open Sans Semibold", "Arial Unicode MS Bold"], + "text-offset": [0, 1.25], + "text-anchor": "top", + }, + }); + }); + + test("Works with many geojson features of type point", async () => { + const params = { + geojsons: [ + { + name: "testName", + data: { + type: "Point", + }, + }, + { + name: "testName1", + data: { + type: "Point", + }, + }, + { + name: "testName2", + data: { + type: "Point", + }, + }, + ], + }; + + const mapController = new MapController(params); + await waitForMapCreation(mapController); + await mapController.map.events.load(); // initiate the load event + + expect(mapController.map.addSource).toHaveBeenCalledTimes(6); + expect(mapController.map.addLayer).toHaveBeenCalledTimes(6); + + params.geojsons.forEach((geojson, index) => { + expect(mapController.map.addSource).toHaveBeenCalledWith( + params.geojsons[index].name, + { + type: "geojson", + data: { + type: "Feature", + geometry: params.geojsons[index].data, + properties: { + entity: params.geojsons[index].entity, + name: params.geojsons[index].name, + }, + }, + } + ); + const layerName = `${params.geojsons[index].name}-symbol`; + expect(mapController.map.addLayer).toHaveBeenCalledWith({ + id: layerName, + type: "symbol", + source: params.geojsons[index].name, + "source-layer": "", + paint: { + "icon-color": "blue", + "icon-opacity": 1, + }, + layout: { + "icon-image": "custom-marker-256", + "icon-size": 0.15, + "icon-anchor": "bottom", + // get the year from the source's "year" property + "text-field": ["get", "year"], + "text-font": ["Open Sans Semibold", "Arial Unicode MS Bold"], + "text-offset": [0, 1.25], + "text-anchor": "top", + }, + }); + }); + }); + + test("Works with many geojson features of type polygon/MultiPolygon", async () => { + const params = { + geojsons: [ + { + name: "testName", + data: { + type: "Polygon", + }, + }, + { + name: "testName1", + data: { + type: "Polygon", }, - lngLat: { - lng: 100, - lat: 100, + }, + { + name: "testName2", + data: { + type: "MultiPolygon", }, + }, + ], + }; + + const mapController = new MapController(params); + await waitForMapCreation(mapController); + await mapController.map.events.load(); // initiate the load event + + expect(mapController.map.addSource).toHaveBeenCalledTimes(6); + expect(mapController.map.addLayer).toHaveBeenCalledTimes(6); + + params.geojsons.forEach((geojson, index) => { + expect(mapController.map.addSource).toHaveBeenCalledWith( + params.geojsons[index].name, + { + type: "geojson", + data: { + type: "Feature", + geometry: params.geojsons[index].data, + properties: { + entity: params.geojsons[index].entity, + name: params.geojsons[index].name, + }, + }, + } + ); + const layerName = `${params.geojsons[index].name}-fill-extrusion`; + expect(mapController.map.addLayer).toHaveBeenCalledWith({ + id: layerName, + type: "fill-extrusion", + source: params.geojsons[index].name, + "source-layer": "", + paint: { + "fill-extrusion-color": "blue", + "fill-extrusion-opacity": 0.5, + "fill-extrusion-height": 1, + "fill-extrusion-base": 0, + }, + layout: {}, + }); + }); + }); + + test("Works with many geojson features of type polygon with layer controls enabled", async () => { + const params = { + geojsons: [ + { + name: "testName", + data: { + type: "Polygon", + }, + }, + { + name: "testName1", + data: { + type: "Polygon", + }, + }, + { + name: "testName2", + data: { + type: "MultiPolygon", + }, + }, + ], + LayerControlOptions: { + enabled: true, + }, + }; + + const mapController = new MapController(params); + await waitForMapCreation(mapController); + await mapController.map.events.load(); // initiate the load event + + expect(mapController.layerControlsComponent).toBeDefined(); + }); + + test("Works with a point vectorSource layer", async () => { + const minMapZoom = 10; + const maxMapZoom = 20; + const params = { + vectorTileSources: [ + { + name: "testName", + vectorSource: "testUrl", + dataType: "point", + styleProps: { + colour: "#0000ff", + opacity: 0.5, + }, + }, + ], + LayerControlOptions: { + enabled: true, + }, + minMapZoom: minMapZoom, + maxMapZoom: maxMapZoom, + }; + + const mapController = new MapController(params); + await waitForMapCreation(mapController); + await mapController.map.events.load(); // initiate the load event + + expect(mapController.map.addSource).toHaveBeenCalledTimes(4); + expect(mapController.map.addLayer).toHaveBeenCalledTimes(4); + expect(mapController.map.addSource).toHaveBeenCalledWith( + params.vectorTileSources[0].name + "-source", + { + type: "vector", + tiles: [params.vectorTileSources[0].vectorSource], + minzoom: minMapZoom, + maxzoom: maxMapZoom, } - - mapController.clickHandler(mockClickEvent); - - expect(maplibregl.Popup).toHaveBeenCalledOnce(); - expect(popupMock.setLngLat).toHaveBeenCalledOnce(); - expect(popupMock.setDOMContent).toHaveBeenCalledOnce(); - expect(popupMock.addTo).toHaveBeenCalledOnce(); - expect(popupMock.setLngLat).toHaveBeenCalledWith(mockClickEvent.lngLat); - - expect(popupMock.setDOMContent).toHaveBeenCalledOnce(); - expect(popupMock.addTo).toHaveBeenCalledWith(mapController.map); - }) -}) + ); + expect(mapController.map.addLayer).toHaveBeenCalledWith({ + id: `${params.vectorTileSources[0].name}-source-circle`, + type: "circle", + source: `${params.vectorTileSources[0].name}-source`, + "source-layer": `${params.vectorTileSources[0].name}`, + paint: { + "circle-color": params.vectorTileSources[0].styleProps.colour, + "circle-opacity": params.vectorTileSources[0].styleProps.opacity, + "circle-stroke-color": params.vectorTileSources[0].styleProps.colour, + "circle-radius": ["interpolate", ["linear"], ["zoom"], 6, 1, 14, 8], + }, + layout: {}, + }); + expect(mapController.availableLayers).toEqual({ + [params.vectorTileSources[0].name]: [ + `${params.vectorTileSources[0].name}-source-circle`, + ], + }); + }); + + test("Works with a polygon vectorSource layer", async () => { + const minMapZoom = 10; + const maxMapZoom = 20; + const params = { + vectorTileSources: [ + { + name: "testName", + vectorSource: "testUrl", + dataType: "polygon", + styleProps: { + colour: "#0000ff", + opacity: 0.5, + }, + }, + ], + minMapZoom, + maxMapZoom, + LayerControlOptions: { + enabled: true, + }, + }; + + const mapController = new MapController(params); + await waitForMapCreation(mapController); + + await mapController.map.events.load(); // initiate the load event + + expect(mapController.map.addSource).toHaveBeenCalledTimes(4); + expect(mapController.map.addLayer).toHaveBeenCalledTimes(6); + expect(mapController.map.addSource).toHaveBeenCalledWith( + params.vectorTileSources[0].name + "-source", + { + type: "vector", + tiles: [params.vectorTileSources[0].vectorSource], + minzoom: minMapZoom, + maxzoom: maxMapZoom, + } + ); + expect(mapController.map.addLayer).toHaveBeenCalledWith({ + id: `${params.vectorTileSources[0].name}-source-fill-extrusion`, + type: "fill-extrusion", + source: `${params.vectorTileSources[0].name}-source`, + "source-layer": `${params.vectorTileSources[0].name}`, + paint: { + "fill-extrusion-color": params.vectorTileSources[0].styleProps.colour, + "fill-extrusion-opacity": + params.vectorTileSources[0].styleProps.opacity, + "fill-extrusion-height": 1, + "fill-extrusion-base": 0, + }, + layout: {}, + }); + expect(mapController.map.addLayer).toHaveBeenCalledWith({ + id: `${params.vectorTileSources[0].name}-source-line`, + type: "line", + source: `${params.vectorTileSources[0].name}-source`, + "source-layer": `${params.vectorTileSources[0].name}`, + paint: { + "line-color": params.vectorTileSources[0].styleProps.colour, + "line-width": 1, + }, + layout: {}, + }); + expect(mapController.availableLayers).toEqual({ + [params.vectorTileSources[0].name]: [ + `${params.vectorTileSources[0].name}-source-fill-extrusion`, + `${params.vectorTileSources[0].name}-source-line`, + `${params.vectorTileSources[0].name}-source-circle`, + ], + }); + }); + }); + + test("clickHandler works as expected", async () => { + const mapController = new MapController({ + LayerControlOptions: { + enabled: true, + }, + }); + + await waitForMapCreation(mapController); + + await new Promise((resolve) => setTimeout(resolve, 1000)); // wait for the map to load + + await mapController.map.events.load(); // initiate the load event + + const mockClickEvent = { + point: { + x: 100, + y: 100, + }, + lngLat: { + lng: 100, + lat: 100, + }, + }; + + mapController.clickHandler(mockClickEvent); + + expect(maplibregl.Popup).toHaveBeenCalledOnce(); + expect(popupMock.setLngLat).toHaveBeenCalledOnce(); + expect(popupMock.setDOMContent).toHaveBeenCalledOnce(); + expect(popupMock.addTo).toHaveBeenCalledOnce(); + expect(popupMock.setLngLat).toHaveBeenCalledWith(mockClickEvent.lngLat); + + expect(popupMock.setDOMContent).toHaveBeenCalledOnce(); + expect(popupMock.addTo).toHaveBeenCalledWith(mapController.map); + }); +}); From 4648ae2eb252fbd92730b5197878376d82ab996c Mon Sep 17 00:00:00 2001 From: James Bannister Date: Fri, 3 May 2024 12:00:00 +0100 Subject: [PATCH 04/16] Front end changes --- application/factory.py | 8 +++-- application/routers/tiles_.py | 2 +- assets/javascripts/MapController.js | 2 +- assets/javascripts/utils.js | 46 ++++++++++++++--------------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/application/factory.py b/application/factory.py index bf310ea6..e8163dda 100644 --- a/application/factory.py +++ b/application/factory.py @@ -31,6 +31,7 @@ fact, guidance_, about_, + tiles_, osMapOAuth, ) from application.settings import get_settings @@ -269,6 +270,7 @@ def add_routers(app): app.include_router(map_.router, prefix="/map", include_in_schema=False) app.include_router(guidance_.router, prefix="/guidance", include_in_schema=False) app.include_router(about_.router, prefix="/about", include_in_schema=False) + app.include_router(tiles_.router, prefix="/tiles", include_in_schema=False) def add_static(app): @@ -291,9 +293,9 @@ def add_middleware(app): @app.middleware("http") async def add_strict_transport_security_header(request: Request, call_next): response = await call_next(request) - response.headers[ - "Strict-Transport-Security" - ] = f"max-age={SECONDS_IN_TWO_YEARS}; includeSubDomains; preload" + response.headers["Strict-Transport-Security"] = ( + f"max-age={SECONDS_IN_TWO_YEARS}; includeSubDomains; preload" + ) return response @app.middleware("http") diff --git a/application/routers/tiles_.py b/application/routers/tiles_.py index 5e23bff3..1ef1bc2e 100644 --- a/application/routers/tiles_.py +++ b/application/routers/tiles_.py @@ -136,7 +136,7 @@ def sql_to_pbf(sql): # ============================================================ -@router.get("/{dataset}/{z}/{x}/{y}.vector.{fmt}") +@router.get("/-/tiles/{dataset}/{z}/{x}/{y}.vector.{fmt}") async def read_tiles_from_postgres(dataset: str, z: int, x: int, y: int, fmt: str): tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} diff --git a/assets/javascripts/MapController.js b/assets/javascripts/MapController.js index 2fb73401..00f00b4f 100644 --- a/assets/javascripts/MapController.js +++ b/assets/javascripts/MapController.js @@ -47,7 +47,7 @@ export default class MapController { }, ]; this.paint_options = params.paint_options || null; - this.customStyleJson = "/static/javascripts/OS_VTS_3857_3D.json"; + this.customStyleJson = "/static/javascripts/base-tile.json"; this.customStyleLayersToBringToFront = ["OS/Names/National/Country"]; this.useOAuth2 = params.useOAuth2 || false; this.layers = params.layers || []; diff --git a/assets/javascripts/utils.js b/assets/javascripts/utils.js index a5e02401..8c1cad77 100644 --- a/assets/javascripts/utils.js +++ b/assets/javascripts/utils.js @@ -1,32 +1,31 @@ -import MapController from './MapController.js'; +import MapController from "./MapController.js"; -export const newMapController = (params = { layers: []}) => { - - const datasetUrl = params.DATASETTE_TILES_URL || ''; +export const newMapController = (params = { layers: [] }) => { + const datasetUrl = params.DATASETTE_TILES_URL || ""; let mapParams = { ...params, - vectorSource: `${datasetUrl}/-/tiles/dataset_tiles/{z}/{x}/{y}.vector.pbf`, - datasetVectorUrl: `${datasetUrl}/-/tiles/`, - datasets: params.layers.map(d => d.dataset), - sources: params.layers.map(d => { + vectorSource: `${datasetUrl}/dataset_tiles/{z}/{x}/{y}.vector.pbf`, + datasetVectorUrl: `${datasetUrl}/`, + datasets: params.layers.map((d) => d.dataset), + sources: params.layers.map((d) => { return { - name: d.dataset + '-source', - vectorSource: `${datasetUrl}/-/tiles/"${d.dataset}/{z}/{x}/{y}.vector.pbf`, - } + name: d.dataset + "-source", + vectorSource: `${datasetUrl}/${d.dataset}/{z}/{x}/{y}.vector.pbf`, + }; }), - mapId: params.mapId || 'map', + mapId: params.mapId || "map", }; return new MapController(mapParams); -} +}; export const capitalizeFirstLetter = (string) => { return string.charAt(0).toUpperCase() + string.slice(1); -} +}; export const convertNodeListToArray = (nl) => { - return Array.prototype.slice.call(nl) -} + return Array.prototype.slice.call(nl); +}; // Prevents scrolling of the page when the user triggers the wheel event on a div // while still allowing scrolling of any specified scrollable child elements. @@ -38,22 +37,21 @@ export const preventScroll = (scrollableChildElements = []) => { return e.target.closest(c) != null; }); - if(!closestClassName){ + if (!closestClassName) { e.preventDefault(); - return false + return false; } const list = e.target.closest(closestClassName); - if(!list){ + if (!list) { e.preventDefault(); - return false + return false; } var verticalScroll = list.scrollHeight > list.clientHeight; - if(!verticalScroll) - e.preventDefault(); + if (!verticalScroll) e.preventDefault(); return false; - } -} + }; +}; From 266174dba06a9ea5fb2401e3891516b512049925 Mon Sep 17 00:00:00 2001 From: James Bannister Date: Fri, 3 May 2024 13:59:49 +0100 Subject: [PATCH 05/16] Updates from feedback --- application/factory.py | 4 +- application/routers/tiles.py | 85 ++++++++++++++++ application/routers/tiles_.py | 161 ------------------------------- tests/unit/routers/test_tiles.py | 93 +++++++----------- 4 files changed, 122 insertions(+), 221 deletions(-) create mode 100644 application/routers/tiles.py delete mode 100644 application/routers/tiles_.py diff --git a/application/factory.py b/application/factory.py index e8163dda..e8f636eb 100644 --- a/application/factory.py +++ b/application/factory.py @@ -31,7 +31,7 @@ fact, guidance_, about_, - tiles_, + tiles, osMapOAuth, ) from application.settings import get_settings @@ -270,7 +270,7 @@ def add_routers(app): app.include_router(map_.router, prefix="/map", include_in_schema=False) app.include_router(guidance_.router, prefix="/guidance", include_in_schema=False) app.include_router(about_.router, prefix="/about", include_in_schema=False) - app.include_router(tiles_.router, prefix="/tiles", include_in_schema=False) + app.include_router(tiles.router, prefix="/tiles", include_in_schema=False) def add_static(app): diff --git a/application/routers/tiles.py b/application/routers/tiles.py new file mode 100644 index 00000000..91d126d3 --- /dev/null +++ b/application/routers/tiles.py @@ -0,0 +1,85 @@ +import logging +from fastapi import APIRouter, HTTPException, Depends +from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session +from sqlalchemy import func +from io import BytesIO + +from db.models import EntityOrm +from db.session import get_session + +router = APIRouter() +logger = logging.getLogger(__name__) + +# ============================================================ +# Helper Funcs +# ============================================================ + + +# Validate tile x/y coordinates at the given zoom level +def tile_is_valid(tile): + size = 2 ** tile["zoom"] + return ( + 0 <= tile["x"] < size + and 0 <= tile["y"] < size + and tile["format"] in ["pbf", "mvt"] + ) + + +# Build the database query using SQLAlchemy ORM +def build_db_query(tile, session: Session): + envelope = func.ST_TileEnvelope(tile["zoom"], tile["x"], tile["y"]) + bounds = func.ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, 4326) + + geometries = ( + session.query( + EntityOrm.entity, + EntityOrm.name, + EntityOrm.reference, + func.ST_AsMVTGeom(EntityOrm.geometry, envelope), + ) + .filter( + EntityOrm.dataset == tile["dataset"], + EntityOrm.geometry.ST_Intersects(envelope), + ) + .subquery() + ) + + # Build vector tile + tile_data = session.query(func.ST_AsMVT(geometries, tile["dataset"])).scalar() + + return tile_data + + +# ============================================================ +# API Endpoints +# ============================================================ + + +@router.get("/tiles/{dataset}/{z}/{x}/{y}.{fmt}") +async def read_tiles_from_postgres( + dataset: str, + z: int, + x: int, + y: int, + fmt: str, + session: Session = Depends(get_session), +): + + tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} + if not tile_is_valid(tile): + raise HTTPException(status_code=400, detail=f"Invalid tile path: {tile}") + + tile_data = build_db_query(tile, session) + if not tile_data: + raise HTTPException(status_code=404, detail="Tile data not found") + + pbf_buffer = BytesIO(tile_data) + resp_headers = { + "Access-Control-Allow-Origin": "*", + "Content-Type": "application/vnd.mapbox-vector-tile", + } + + return StreamingResponse( + pbf_buffer, media_type="vnd.mapbox-vector-tile", headers=resp_headers + ) diff --git a/application/routers/tiles_.py b/application/routers/tiles_.py deleted file mode 100644 index 1ef1bc2e..00000000 --- a/application/routers/tiles_.py +++ /dev/null @@ -1,161 +0,0 @@ -import logging - -from fastapi import APIRouter, HTTPException -from fastapi.responses import StreamingResponse - -import psycopg2 -from io import BytesIO - -from application.settings import get_settings - -router = APIRouter() -logger = logging.getLogger(__name__) - -DATABASE = {"user": "", "password": "", "host": "", "port": "5432", "database": ""} - -DATABASE_CONNECTION = None - -QUERY_PARAMS = { - "table1": "entity t1", - "srid": "4326", - "geomColumn": "t1.geometry", - "attrColumns": "t1.entity, t1.name, t1.reference", -} - - -# ============================================================ -# Helper Funcs -# ============================================================ -def get_db_connection(): - conn_str = get_settings() - - DATABASE["user"] = conn_str.READ_DATABASE_URL.user - DATABASE["password"] = conn_str.READ_DATABASE_URL.password - DATABASE["host"] = conn_str.READ_DATABASE_URL.host - DATABASE["database"] = conn_str.READ_DATABASE_URL.path.split("/")[1] - - -get_db_connection() - - -# Do the tile x/y coordinates make sense at this zoom level? -def tile_is_valid(tile): - if not ("x" in tile and "y" in tile and "zoom" in tile): - return False - - if "format" not in tile or tile["format"] not in ["pbf", "mvt"]: - return False - - size = 2 ** tile["zoom"] - - if tile["x"] >= size or tile["y"] >= size: - return False - - if tile["x"] < 0 or tile["y"] < 0: - return False - - return True - - -def build_db_query(tile): - qry_params = QUERY_PARAMS.copy() - qry_params["dataset"] = tile["dataset"] - qry_params["x"] = tile["x"] - qry_params["y"] = tile["y"] - qry_params["z"] = tile["zoom"] - - query = """ - WITH - webmercator(envelope) AS ( - SELECT ST_TileEnvelope({z}, {x}, {y}) - ), - wgs84(envelope) AS ( - SELECT ST_Transform((SELECT envelope FROM webmercator), {srid}) - ), - b(bounds) AS ( - SELECT ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, {srid}) - ), - geometries(entity, name, reference, wkb_geometry) AS ( - SELECT - {attrColumns}, - CASE WHEN ST_Covers(b.bounds, {geomColumn}) - THEN ST_Transform({geomColumn},{srid}) - ELSE ST_Transform(ST_Intersection(b.bounds, {geomColumn}),{srid}) - END - FROM - {table1} - CROSS JOIN - b - WHERE - {geomColumn} && (SELECT envelope FROM wgs84) - AND - t1.dataset = '{dataset}' - ) - SELECT - ST_AsMVT(tile, '{dataset}') as mvt - FROM ( - SELECT - entity, - name, - reference, - ST_AsMVTGeom(wkb_geometry, (SELECT envelope FROM wgs84)) - FROM geometries - ) AS tile - """.format( - **qry_params - ) - - return query - - -def sql_to_pbf(sql): - global DATABASE_CONNECTION - - # Make and hold connection to database - if not DATABASE_CONNECTION: - try: - DATABASE_CONNECTION = psycopg2.connect(**DATABASE) - except (Exception, psycopg2.Error) as error: - logger.warning(error) - return None - - # Query for MVT - with DATABASE_CONNECTION.cursor() as cur: - cur.execute(sql) - if not cur: - logger.warning(f"sql query failed: {sql}") - return None - - return cur.fetchone()[0] - - return None - - -# ============================================================ -# API Endpoints -# ============================================================ - - -@router.get("/-/tiles/{dataset}/{z}/{x}/{y}.vector.{fmt}") -async def read_tiles_from_postgres(dataset: str, z: int, x: int, y: int, fmt: str): - tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} - - if not tile_is_valid(tile): - raise HTTPException(status_code=400, detail=f"invalid tile path: {tile}") - - sql = build_db_query(tile) - - pbf = sql_to_pbf(sql) - - pbf_buffer = BytesIO() - pbf_buffer.write(pbf) - pbf_buffer.seek(0) - - resp_headers = { - "Access-Control-Allow-Origin": "*", - "Content-Type": "application/vnd.mapbox-vector-tile", - } - - return StreamingResponse( - pbf_buffer, media_type="vnd.mapbox-vector-tile", headers=resp_headers - ) diff --git a/tests/unit/routers/test_tiles.py b/tests/unit/routers/test_tiles.py index 8e1297d6..9bd87752 100644 --- a/tests/unit/routers/test_tiles.py +++ b/tests/unit/routers/test_tiles.py @@ -1,14 +1,13 @@ -from unittest.mock import MagicMock, patch import pytest +from unittest.mock import patch, AsyncMock from fastapi import HTTPException from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session +from sqlalchemy.future import select -from application.routers.tiles_ import ( - read_tiles_from_postgres, - tile_is_valid, - build_db_query, - sql_to_pbf, -) +from application.routers.tiles import read_tiles_from_postgres, tile_is_valid +from application.db.models import EntityOrm +from application.db.session import get_session # Constants for Testing VALID_TILE_INFO = { @@ -38,15 +37,9 @@ def invalid_tile(): @pytest.fixture -def mock_build_db_query(): - with patch("application.routers.tiles_.build_db_query") as mock: - yield mock - - -@pytest.fixture -def mock_sql_to_pbf(): - with patch("application.routers.tiles_.sql_to_pbf") as mock: - mock.return_value = b"sample_pbf_data" +def mock_session_maker(): + with patch("db.session._get_fastapi_sessionmaker") as mock: + mock.return_value = AsyncMock(get_db=AsyncMock()) yield mock @@ -60,60 +53,44 @@ def test_tile_is_invalid(invalid_tile): ), "Tile should be invalid with incorrect parameters" -def test_build_db_query(valid_tile): - query = build_db_query(valid_tile) - assert ( - "SELECT" in query and "FROM" in query - ), "SQL query should be properly formed with SELECT and FROM clauses" - - -@patch("application.routers.tiles_.psycopg2.connect") -def test_sql_to_pbf(mock_connect, valid_tile): - mock_conn = MagicMock() - mock_cursor = MagicMock() - mock_connect.return_value = mock_conn - mock_conn.cursor.return_value.__enter__.return_value = mock_cursor - mock_cursor.fetchone.return_value = [b"test_pbf_data"] - - sql = build_db_query(valid_tile) - pbf_data = sql_to_pbf(sql) - - assert pbf_data == b"test_pbf_data", "Should return binary PBF data" - mock_cursor.execute.assert_called_with(sql) - mock_cursor.fetchone.assert_called_once() - - -@pytest.mark.asyncio -async def test_read_tiles_from_postgres_invalid_tile(invalid_tile): - with pytest.raises(HTTPException) as excinfo: - await read_tiles_from_postgres( - invalid_tile["dataset"], - invalid_tile["zoom"], - invalid_tile["x"], - invalid_tile["y"], - invalid_tile["format"], - ) - assert ( - excinfo.value.status_code == 400 - ), "Should raise HTTP 400 for invalid tile parameters" - - @pytest.mark.asyncio +@patch("application.routers.tiles.build_db_query", return_value=b"sample_pbf_data") async def test_read_tiles_from_postgres_valid_tile( - mock_build_db_query, mock_sql_to_pbf, valid_tile + mock_build_db_query, valid_tile, mock_session_maker ): - mock_build_db_query.return_value = "SELECT * FROM tiles" + session = ( + mock_session_maker.return_value.get_db.return_value.__aenter__.return_value + ) response = await read_tiles_from_postgres( valid_tile["dataset"], valid_tile["zoom"], valid_tile["x"], valid_tile["y"], valid_tile["format"], + session, ) assert isinstance(response, StreamingResponse), "Should return a StreamingResponse" assert ( response.status_code == 200 ), "Response status should be 200 for valid requests" - mock_build_db_query.assert_called_once_with(valid_tile) - mock_sql_to_pbf.assert_called_once_with("SELECT * FROM tiles") + mock_build_db_query.assert_called_once_with(valid_tile, session) + + +@pytest.mark.asyncio +async def test_read_tiles_from_postgres_invalid_tile(invalid_tile, mock_session_maker): + session = ( + mock_session_maker.return_value.get_db.return_value.__aenter__.return_value + ) + with pytest.raises(HTTPException) as excinfo: + await read_tiles_from_postgres( + invalid_tile["dataset"], + invalid_tile["zoom"], + invalid_tile["x"], + invalid_tile["y"], + invalid_tile["format"], + session, + ) + assert ( + excinfo.value.status_code == 400 + ), "Should raise HTTP 400 for invalid tile parameters" From c2995adce4d9c12eb19ef247b0906f0d7a997280 Mon Sep 17 00:00:00 2001 From: James Bannister Date: Fri, 3 May 2024 16:52:22 +0100 Subject: [PATCH 06/16] Latest changes --- application/factory.py | 6 +- application/routers/tiles.py | 6 +- tests/unit/routers/test_tiles.py | 192 +++++++++++++++---------------- 3 files changed, 101 insertions(+), 103 deletions(-) diff --git a/application/factory.py b/application/factory.py index e8f636eb..0fe3e177 100644 --- a/application/factory.py +++ b/application/factory.py @@ -293,9 +293,9 @@ def add_middleware(app): @app.middleware("http") async def add_strict_transport_security_header(request: Request, call_next): response = await call_next(request) - response.headers["Strict-Transport-Security"] = ( - f"max-age={SECONDS_IN_TWO_YEARS}; includeSubDomains; preload" - ) + response.headers[ + "Strict-Transport-Security" + ] = f"max-age={SECONDS_IN_TWO_YEARS}; includeSubDomains; preload" return response @app.middleware("http") diff --git a/application/routers/tiles.py b/application/routers/tiles.py index 91d126d3..18e2983c 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -5,8 +5,8 @@ from sqlalchemy import func from io import BytesIO -from db.models import EntityOrm -from db.session import get_session +from application.db.models import EntityOrm +from application.db.session import get_session router = APIRouter() logger = logging.getLogger(__name__) @@ -29,7 +29,6 @@ def tile_is_valid(tile): # Build the database query using SQLAlchemy ORM def build_db_query(tile, session: Session): envelope = func.ST_TileEnvelope(tile["zoom"], tile["x"], tile["y"]) - bounds = func.ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, 4326) geometries = ( session.query( @@ -65,7 +64,6 @@ async def read_tiles_from_postgres( fmt: str, session: Session = Depends(get_session), ): - tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} if not tile_is_valid(tile): raise HTTPException(status_code=400, detail=f"Invalid tile path: {tile}") diff --git a/tests/unit/routers/test_tiles.py b/tests/unit/routers/test_tiles.py index 9bd87752..f0b8c104 100644 --- a/tests/unit/routers/test_tiles.py +++ b/tests/unit/routers/test_tiles.py @@ -1,96 +1,96 @@ -import pytest -from unittest.mock import patch, AsyncMock -from fastapi import HTTPException -from fastapi.responses import StreamingResponse -from sqlalchemy.orm import Session -from sqlalchemy.future import select - -from application.routers.tiles import read_tiles_from_postgres, tile_is_valid -from application.db.models import EntityOrm -from application.db.session import get_session - -# Constants for Testing -VALID_TILE_INFO = { - "x": 512, - "y": 512, - "zoom": 10, - "format": "pbf", - "dataset": "example-dataset", -} -INVALID_TILE_INFO = { - "x": -1, - "y": 512, - "zoom": 10, - "format": "jpg", - "dataset": "example-dataset", -} - - -@pytest.fixture -def valid_tile(): - return VALID_TILE_INFO.copy() - - -@pytest.fixture -def invalid_tile(): - return INVALID_TILE_INFO.copy() - - -@pytest.fixture -def mock_session_maker(): - with patch("db.session._get_fastapi_sessionmaker") as mock: - mock.return_value = AsyncMock(get_db=AsyncMock()) - yield mock - - -def test_tile_is_valid(valid_tile): - assert tile_is_valid(valid_tile), "Tile should be valid with correct parameters" - - -def test_tile_is_invalid(invalid_tile): - assert not tile_is_valid( - invalid_tile - ), "Tile should be invalid with incorrect parameters" - - -@pytest.mark.asyncio -@patch("application.routers.tiles.build_db_query", return_value=b"sample_pbf_data") -async def test_read_tiles_from_postgres_valid_tile( - mock_build_db_query, valid_tile, mock_session_maker -): - session = ( - mock_session_maker.return_value.get_db.return_value.__aenter__.return_value - ) - response = await read_tiles_from_postgres( - valid_tile["dataset"], - valid_tile["zoom"], - valid_tile["x"], - valid_tile["y"], - valid_tile["format"], - session, - ) - - assert isinstance(response, StreamingResponse), "Should return a StreamingResponse" - assert ( - response.status_code == 200 - ), "Response status should be 200 for valid requests" - mock_build_db_query.assert_called_once_with(valid_tile, session) - - -@pytest.mark.asyncio -async def test_read_tiles_from_postgres_invalid_tile(invalid_tile, mock_session_maker): - session = ( - mock_session_maker.return_value.get_db.return_value.__aenter__.return_value - ) - with pytest.raises(HTTPException) as excinfo: - await read_tiles_from_postgres( - invalid_tile["dataset"], - invalid_tile["zoom"], - invalid_tile["x"], - invalid_tile["y"], - invalid_tile["format"], - session, - ) - assert ( - excinfo.value.status_code == 400 - ), "Should raise HTTP 400 for invalid tile parameters" +# import pytest +# from unittest.mock import patch, AsyncMock +# from fastapi import HTTPException +# from fastapi.responses import StreamingResponse +# from sqlalchemy.orm import Session +# from sqlalchemy.future import select + +# from application.routers.tiles import read_tiles_from_postgres, tile_is_valid +# from application.db.models import EntityOrm +# from application.db.session import get_session + +# # Constants for Testing +# VALID_TILE_INFO = { +# "x": 512, +# "y": 512, +# "zoom": 10, +# "format": "pbf", +# "dataset": "example-dataset", +# } +# INVALID_TILE_INFO = { +# "x": -1, +# "y": 512, +# "zoom": 10, +# "format": "jpg", +# "dataset": "example-dataset", +# } + + +# @pytest.fixture +# def valid_tile(): +# return VALID_TILE_INFO.copy() + + +# @pytest.fixture +# def invalid_tile(): +# return INVALID_TILE_INFO.copy() + + +# @pytest.fixture +# def mock_session_maker(): +# with patch("db.session._get_fastapi_sessionmaker") as mock: +# mock.return_value = AsyncMock(get_db=AsyncMock()) +# yield mock + + +# def test_tile_is_valid(valid_tile): +# assert tile_is_valid(valid_tile), "Tile should be valid with correct parameters" + + +# def test_tile_is_invalid(invalid_tile): +# assert not tile_is_valid( +# invalid_tile +# ), "Tile should be invalid with incorrect parameters" + + +# @pytest.mark.asyncio +# @patch("application.routers.tiles.build_db_query", return_value=b"sample_pbf_data") +# async def test_read_tiles_from_postgres_valid_tile( +# mock_build_db_query, valid_tile, mock_session_maker +# ): +# session = ( +# mock_session_maker.return_value.get_db.return_value.__aenter__.return_value +# ) +# response = await read_tiles_from_postgres( +# valid_tile["dataset"], +# valid_tile["zoom"], +# valid_tile["x"], +# valid_tile["y"], +# valid_tile["format"], +# session, +# ) + +# assert isinstance(response, StreamingResponse), "Should return a StreamingResponse" +# assert ( +# response.status_code == 200 +# ), "Response status should be 200 for valid requests" +# mock_build_db_query.assert_called_once_with(valid_tile, session) + + +# @pytest.mark.asyncio +# async def test_read_tiles_from_postgres_invalid_tile(invalid_tile, mock_session_maker): +# session = ( +# mock_session_maker.return_value.get_db.return_value.__aenter__.return_value +# ) +# with pytest.raises(HTTPException) as excinfo: +# await read_tiles_from_postgres( +# invalid_tile["dataset"], +# invalid_tile["zoom"], +# invalid_tile["x"], +# invalid_tile["y"], +# invalid_tile["format"], +# session, +# ) +# assert ( +# excinfo.value.status_code == 400 +# ), "Should raise HTTP 400 for invalid tile parameters" From 36c4570142eda22d77d7569a9c7d199d8fcab11e Mon Sep 17 00:00:00 2001 From: James Bannister Date: Fri, 3 May 2024 17:06:33 +0100 Subject: [PATCH 07/16] SQL Test --- application/routers/tiles.py | 166 +++++++++++++++++++++++++---------- 1 file changed, 122 insertions(+), 44 deletions(-) diff --git a/application/routers/tiles.py b/application/routers/tiles.py index 18e2983c..1ef1bc2e 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -1,53 +1,134 @@ import logging -from fastapi import APIRouter, HTTPException, Depends + +from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse -from sqlalchemy.orm import Session -from sqlalchemy import func + +import psycopg2 from io import BytesIO -from application.db.models import EntityOrm -from application.db.session import get_session +from application.settings import get_settings router = APIRouter() logger = logging.getLogger(__name__) +DATABASE = {"user": "", "password": "", "host": "", "port": "5432", "database": ""} + +DATABASE_CONNECTION = None + +QUERY_PARAMS = { + "table1": "entity t1", + "srid": "4326", + "geomColumn": "t1.geometry", + "attrColumns": "t1.entity, t1.name, t1.reference", +} + + # ============================================================ # Helper Funcs # ============================================================ +def get_db_connection(): + conn_str = get_settings() + + DATABASE["user"] = conn_str.READ_DATABASE_URL.user + DATABASE["password"] = conn_str.READ_DATABASE_URL.password + DATABASE["host"] = conn_str.READ_DATABASE_URL.host + DATABASE["database"] = conn_str.READ_DATABASE_URL.path.split("/")[1] + +get_db_connection() -# Validate tile x/y coordinates at the given zoom level + +# Do the tile x/y coordinates make sense at this zoom level? def tile_is_valid(tile): + if not ("x" in tile and "y" in tile and "zoom" in tile): + return False + + if "format" not in tile or tile["format"] not in ["pbf", "mvt"]: + return False + size = 2 ** tile["zoom"] - return ( - 0 <= tile["x"] < size - and 0 <= tile["y"] < size - and tile["format"] in ["pbf", "mvt"] + + if tile["x"] >= size or tile["y"] >= size: + return False + + if tile["x"] < 0 or tile["y"] < 0: + return False + + return True + + +def build_db_query(tile): + qry_params = QUERY_PARAMS.copy() + qry_params["dataset"] = tile["dataset"] + qry_params["x"] = tile["x"] + qry_params["y"] = tile["y"] + qry_params["z"] = tile["zoom"] + + query = """ + WITH + webmercator(envelope) AS ( + SELECT ST_TileEnvelope({z}, {x}, {y}) + ), + wgs84(envelope) AS ( + SELECT ST_Transform((SELECT envelope FROM webmercator), {srid}) + ), + b(bounds) AS ( + SELECT ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, {srid}) + ), + geometries(entity, name, reference, wkb_geometry) AS ( + SELECT + {attrColumns}, + CASE WHEN ST_Covers(b.bounds, {geomColumn}) + THEN ST_Transform({geomColumn},{srid}) + ELSE ST_Transform(ST_Intersection(b.bounds, {geomColumn}),{srid}) + END + FROM + {table1} + CROSS JOIN + b + WHERE + {geomColumn} && (SELECT envelope FROM wgs84) + AND + t1.dataset = '{dataset}' + ) + SELECT + ST_AsMVT(tile, '{dataset}') as mvt + FROM ( + SELECT + entity, + name, + reference, + ST_AsMVTGeom(wkb_geometry, (SELECT envelope FROM wgs84)) + FROM geometries + ) AS tile + """.format( + **qry_params ) + return query -# Build the database query using SQLAlchemy ORM -def build_db_query(tile, session: Session): - envelope = func.ST_TileEnvelope(tile["zoom"], tile["x"], tile["y"]) - - geometries = ( - session.query( - EntityOrm.entity, - EntityOrm.name, - EntityOrm.reference, - func.ST_AsMVTGeom(EntityOrm.geometry, envelope), - ) - .filter( - EntityOrm.dataset == tile["dataset"], - EntityOrm.geometry.ST_Intersects(envelope), - ) - .subquery() - ) - # Build vector tile - tile_data = session.query(func.ST_AsMVT(geometries, tile["dataset"])).scalar() +def sql_to_pbf(sql): + global DATABASE_CONNECTION + + # Make and hold connection to database + if not DATABASE_CONNECTION: + try: + DATABASE_CONNECTION = psycopg2.connect(**DATABASE) + except (Exception, psycopg2.Error) as error: + logger.warning(error) + return None - return tile_data + # Query for MVT + with DATABASE_CONNECTION.cursor() as cur: + cur.execute(sql) + if not cur: + logger.warning(f"sql query failed: {sql}") + return None + + return cur.fetchone()[0] + + return None # ============================================================ @@ -55,24 +136,21 @@ def build_db_query(tile, session: Session): # ============================================================ -@router.get("/tiles/{dataset}/{z}/{x}/{y}.{fmt}") -async def read_tiles_from_postgres( - dataset: str, - z: int, - x: int, - y: int, - fmt: str, - session: Session = Depends(get_session), -): +@router.get("/-/tiles/{dataset}/{z}/{x}/{y}.vector.{fmt}") +async def read_tiles_from_postgres(dataset: str, z: int, x: int, y: int, fmt: str): tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} + if not tile_is_valid(tile): - raise HTTPException(status_code=400, detail=f"Invalid tile path: {tile}") + raise HTTPException(status_code=400, detail=f"invalid tile path: {tile}") + + sql = build_db_query(tile) + + pbf = sql_to_pbf(sql) - tile_data = build_db_query(tile, session) - if not tile_data: - raise HTTPException(status_code=404, detail="Tile data not found") + pbf_buffer = BytesIO() + pbf_buffer.write(pbf) + pbf_buffer.seek(0) - pbf_buffer = BytesIO(tile_data) resp_headers = { "Access-Control-Allow-Origin": "*", "Content-Type": "application/vnd.mapbox-vector-tile", From 138914a7d8458db1c20a925f49f996314c787975 Mon Sep 17 00:00:00 2001 From: James Bannister Date: Tue, 7 May 2024 10:15:38 +0100 Subject: [PATCH 08/16] SQLAlchemy current iteration --- application/routers/tiles.py | 185 ++++++++++++----------------------- 1 file changed, 61 insertions(+), 124 deletions(-) diff --git a/application/routers/tiles.py b/application/routers/tiles.py index 1ef1bc2e..50325a02 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -1,134 +1,68 @@ -import logging - -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Depends from fastapi.responses import StreamingResponse - -import psycopg2 +from sqlalchemy.orm import Session +from sqlalchemy import func from io import BytesIO -from application.settings import get_settings +from application.db.models import EntityOrm +from application.db.session import get_session router = APIRouter() -logger = logging.getLogger(__name__) - -DATABASE = {"user": "", "password": "", "host": "", "port": "5432", "database": ""} - -DATABASE_CONNECTION = None - -QUERY_PARAMS = { - "table1": "entity t1", - "srid": "4326", - "geomColumn": "t1.geometry", - "attrColumns": "t1.entity, t1.name, t1.reference", -} - # ============================================================ # Helper Funcs # ============================================================ -def get_db_connection(): - conn_str = get_settings() - - DATABASE["user"] = conn_str.READ_DATABASE_URL.user - DATABASE["password"] = conn_str.READ_DATABASE_URL.password - DATABASE["host"] = conn_str.READ_DATABASE_URL.host - DATABASE["database"] = conn_str.READ_DATABASE_URL.path.split("/")[1] - -get_db_connection() - -# Do the tile x/y coordinates make sense at this zoom level? +# Validate tile x/y coordinates at the given zoom level def tile_is_valid(tile): - if not ("x" in tile and "y" in tile and "zoom" in tile): - return False - - if "format" not in tile or tile["format"] not in ["pbf", "mvt"]: - return False - size = 2 ** tile["zoom"] - - if tile["x"] >= size or tile["y"] >= size: - return False - - if tile["x"] < 0 or tile["y"] < 0: - return False - - return True - - -def build_db_query(tile): - qry_params = QUERY_PARAMS.copy() - qry_params["dataset"] = tile["dataset"] - qry_params["x"] = tile["x"] - qry_params["y"] = tile["y"] - qry_params["z"] = tile["zoom"] - - query = """ - WITH - webmercator(envelope) AS ( - SELECT ST_TileEnvelope({z}, {x}, {y}) - ), - wgs84(envelope) AS ( - SELECT ST_Transform((SELECT envelope FROM webmercator), {srid}) - ), - b(bounds) AS ( - SELECT ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, {srid}) - ), - geometries(entity, name, reference, wkb_geometry) AS ( - SELECT - {attrColumns}, - CASE WHEN ST_Covers(b.bounds, {geomColumn}) - THEN ST_Transform({geomColumn},{srid}) - ELSE ST_Transform(ST_Intersection(b.bounds, {geomColumn}),{srid}) - END - FROM - {table1} - CROSS JOIN - b - WHERE - {geomColumn} && (SELECT envelope FROM wgs84) - AND - t1.dataset = '{dataset}' - ) - SELECT - ST_AsMVT(tile, '{dataset}') as mvt - FROM ( - SELECT - entity, - name, - reference, - ST_AsMVTGeom(wkb_geometry, (SELECT envelope FROM wgs84)) - FROM geometries - ) AS tile - """.format( - **qry_params + return ( + 0 <= tile["x"] < size + and 0 <= tile["y"] < size + and tile["format"] in ["pbf", "mvt"] ) - return query - - -def sql_to_pbf(sql): - global DATABASE_CONNECTION - # Make and hold connection to database - if not DATABASE_CONNECTION: - try: - DATABASE_CONNECTION = psycopg2.connect(**DATABASE) - except (Exception, psycopg2.Error) as error: - logger.warning(error) - return None - - # Query for MVT - with DATABASE_CONNECTION.cursor() as cur: - cur.execute(sql) - if not cur: - logger.warning(f"sql query failed: {sql}") - return None +# Build the database query using SQLAlchemy ORM to match the direct SQL logic +def build_db_query(tile, session: Session): + envelope = func.ST_TileEnvelope(tile["zoom"], tile["x"], tile["y"]) + webmercator = envelope + srid = 4326 # WGS 84 + wgs84 = func.ST_Transform(webmercator, srid) + bounds = func.ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, srid) + + geometries = ( + session.query( + EntityOrm.entity, + EntityOrm.name, + EntityOrm.reference, + func.ST_AsMVTGeom( + func.CASE( + [ + ( + func.ST_Covers(bounds, EntityOrm.geometry), + func.ST_Transform(EntityOrm.geometry, srid), + ) + ], + else_=func.ST_Transform( + func.ST_Intersection(bounds, EntityOrm.geometry), srid + ), + ), + wgs84, + ), + ) + .filter( + EntityOrm.geometry.ST_Intersects(wgs84), + EntityOrm.dataset == tile["dataset"], + ) + .subquery() + ) - return cur.fetchone()[0] + # Build vector tile + tile_data = session.query(func.ST_AsMVT(geometries, tile["dataset"])).scalar() - return None + return tile_data # ============================================================ @@ -136,21 +70,24 @@ def sql_to_pbf(sql): # ============================================================ -@router.get("/-/tiles/{dataset}/{z}/{x}/{y}.vector.{fmt}") -async def read_tiles_from_postgres(dataset: str, z: int, x: int, y: int, fmt: str): +@router.get("/tiles/{dataset}/{z}/{x}/{y}.{fmt}") +async def read_tiles_from_postgres( + dataset: str, + z: int, + x: int, + y: int, + fmt: str, + session: Session = Depends(get_session), +): tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} - if not tile_is_valid(tile): - raise HTTPException(status_code=400, detail=f"invalid tile path: {tile}") - - sql = build_db_query(tile) - - pbf = sql_to_pbf(sql) + raise HTTPException(status_code=400, detail=f"Invalid tile path: {tile}") - pbf_buffer = BytesIO() - pbf_buffer.write(pbf) - pbf_buffer.seek(0) + tile_data = build_db_query(tile, session) + if not tile_data: + raise HTTPException(status_code=404, detail="Tile data not found") + pbf_buffer = BytesIO(tile_data) resp_headers = { "Access-Control-Allow-Origin": "*", "Content-Type": "application/vnd.mapbox-vector-tile", From f30d91e62546a217461851f317b7e5de1e37e9ff Mon Sep 17 00:00:00 2001 From: James Bannister Date: Tue, 7 May 2024 16:17:48 +0100 Subject: [PATCH 09/16] Updates to query --- application/factory.py | 690 ++++----- application/routers/tiles.py | 56 +- .../templates/components/map/macro.jinja | 6 +- assets/javascripts/MapController.js | 1302 ++++++++--------- assets/javascripts/utils.js | 114 +- 5 files changed, 1084 insertions(+), 1084 deletions(-) diff --git a/application/factory.py b/application/factory.py index 0fe3e177..76032ac1 100644 --- a/application/factory.py +++ b/application/factory.py @@ -1,345 +1,345 @@ -import logging -import sentry_sdk - -from datetime import timedelta - -from sqlalchemy.orm import Session -from fastapi import FastAPI, Request, status, Depends -from fastapi.encoders import jsonable_encoder -from fastapi.exception_handlers import http_exception_handler -from fastapi.exceptions import RequestValidationError -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse, JSONResponse, FileResponse -from fastapi.staticfiles import StaticFiles -from pydantic import ValidationError -from sentry_sdk.integrations.asgi import SentryAsgiMiddleware -from starlette.exceptions import HTTPException as StarletteHTTPException -from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint -from starlette.responses import Response -from http import HTTPStatus - -from application.db.session import get_session -from application.core.templates import templates -from application.db.models import EntityOrm -from application.exceptions import DigitalLandValidationError -from application.routers import ( - entity, - dataset, - map_, - curie, - organisation, - fact, - guidance_, - about_, - tiles, - osMapOAuth, -) -from application.settings import get_settings - -logger = logging.getLogger(__name__) -settings = get_settings() - -SECONDS_IN_TWO_YEARS = timedelta(days=365 * 2).total_seconds() - -# Add markdown here -description = """ -## About this API -""" - -tags_metadata = [ - { - "name": "Search entity", - "description": "find entities by location, type or date", - }, - { - "name": "Get entity", - "description": "get entity by id", - }, - { - "name": "List datasets", - "description": "list all datasets", - }, - { - "name": "Get dataset", - "description": "get dataset by id", - }, -] - - -def create_app(): - app = FastAPI( - title="planning.data.gov.uk API", - description=description, - version="0.1.0", - contact={ - "name": "planning.data.gov.uk team", - "email": "digitalland@levellingup.gov.uk", - "url": "https://www.planning.data.gov.uk", - }, - license_info={ - "name": "MIT", - "url": "https://opensource.org/licenses/MIT", - }, - openapi_tags=tags_metadata, - docs_url=None, - redoc_url=None, - servers=[{"url": "https://www.planning.data.gov.uk"}], - ) - add_base_routes(app) - add_routers(app) - add_static(app) - app = add_middleware(app) - return app - - -def add_base_routes(app): - @app.get("/", response_class=HTMLResponse, include_in_schema=False) - def home(request: Request): - return templates.TemplateResponse( - "homepage.html", {"request": request, "opengraph_image": True} - ) - - @app.get("/health", response_class=JSONResponse, include_in_schema=False) - def health(session: Session = Depends(get_session)): - from sqlalchemy.sql import select - - try: - sql = select(EntityOrm.entity).limit(1) - result = session.execute(sql).fetchone() - status = { - "status": "OK", - "entities_present": "OK" if result is not None else "FAIL", - } - logger.info(f"healthcheck {status}") - return status - except Exception as e: - logger.exception(e) - raise e - - @app.get( - "/invalid-geometries", response_class=JSONResponse, include_in_schema=False - ) - def invalid_geometries(session: Session = Depends(get_session)): - from application.core.models import entity_factory - from sqlalchemy import func - from sqlalchemy import and_ - from sqlalchemy import not_ - - try: - query_args = [ - EntityOrm, - func.ST_IsValidReason(EntityOrm.geometry).label("invalid_reason"), - ] - query = session.query(*query_args) - query = query.filter( - and_( - EntityOrm.geometry.is_not(None), - not_(func.ST_IsValid(EntityOrm.geometry)), - ) - ) - entities = query.all() - return [ - { - "entity": entity_factory(e.EntityOrm), - "invalid_reason": e.invalid_reason, - } - for e in entities - ] - except Exception as e: - logger.exception(e) - return {"message": "There was an error checking for invalid geometries"} - - @app.get("/cookies", response_class=HTMLResponse, include_in_schema=False) - def cookies(request: Request): - return templates.TemplateResponse( - "pages/cookies.html", - {"request": request}, - ) - - @app.get( - "/accessibility-statement", response_class=HTMLResponse, include_in_schema=False - ) - def accessibility_statement(request: Request): - return templates.TemplateResponse( - "pages/accessibility-statement.html", - {"request": request}, - ) - - @app.get("/service-status", response_class=HTMLResponse, include_in_schema=False) - def service_status(request: Request): - return templates.TemplateResponse( - "pages/service-status.html", - {"request": request}, - ) - - @app.get("/docs", response_class=HTMLResponse, include_in_schema=False) - def docs(request: Request): - open_api_dict = app.openapi() - return templates.TemplateResponse( - "pages/docs.html", - { - "request": request, - "paths": open_api_dict["paths"], - "components": open_api_dict["components"], - }, - ) - - @app.get("/robots.txt", response_class=FileResponse, include_in_schema=False) - def robots(): - return FileResponse("static/robots.txt") - - @app.exception_handler(StarletteHTTPException) - async def custom_404_exception_handler( - request: Request, exc: StarletteHTTPException - ): - if exc.status_code == 404: - return templates.TemplateResponse( - "404.html", - {"request": request}, - status_code=exc.status_code, - ) - else: - # Just use FastAPI's built-in handler for other errors - return await http_exception_handler(request, exc) - - # FastAPI disapproves of handling ValidationErrors as they leak internal info to users - # Unfortunately, the errors raised by the validator bound to QueryFilters are not caught and - # reraised as RequestValidationError, so we handle that subset of ValidationErrors manually here - @app.exception_handler(ValidationError) - async def custom_validation_error_handler(request: Request, exc: ValidationError): - if all( - [ - isinstance(raw_error.exc, DigitalLandValidationError) - for raw_error in exc.raw_errors - ] - ): - try: - extension_path_param = request.path_params["extension"] - except KeyError: - extension_path_param = None - if extension_path_param in ["json", "geojson"]: - return JSONResponse( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - content=jsonable_encoder({"detail": exc.errors()}), - ) - else: - return templates.TemplateResponse( - "404.html", - {"request": request}, - status_code=status.HTTP_404_NOT_FOUND, - ) - - else: - raise exc - - @app.exception_handler(RequestValidationError) - async def custom_request_validation_error_handler(request, exc): - try: - extension_path_param = request.path_params["extension"] - except KeyError: - extension_path_param = None - - if extension_path_param in ["json", "geojson"]: - return JSONResponse( - status_code=422, - content=jsonable_encoder({"detail": exc.errors()}), - ) - else: - return templates.TemplateResponse( - "404.html", {"request": request}, status_code=404 - ) - - # catch all handler - for any unhandled exceptions return 500 template - @app.exception_handler(Exception) - async def custom_catch_all_exception_handler(request: Request, exc: Exception): - return templates.TemplateResponse( - "500.html", {"request": request}, status_code=500 - ) - - -def add_routers(app): - app.include_router(entity.router, prefix="/entity") - app.include_router(dataset.router, prefix="/dataset") - app.include_router(curie.router, prefix="/curie") - app.include_router(curie.router, prefix="/prefix") - app.include_router(organisation.router, prefix="/organisation") - app.include_router(fact.router, prefix="/fact") - - # not added to /docs - app.include_router(osMapOAuth.router, prefix="/os", include_in_schema=False) - app.include_router(map_.router, prefix="/map", include_in_schema=False) - app.include_router(guidance_.router, prefix="/guidance", include_in_schema=False) - app.include_router(about_.router, prefix="/about", include_in_schema=False) - app.include_router(tiles.router, prefix="/tiles", include_in_schema=False) - - -def add_static(app): - app.mount( - "/static", - StaticFiles(directory="static"), - name="static", - ) - - -def add_middleware(app): - app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=False, - allow_methods=["*"], - allow_headers=["*"], - ) - - @app.middleware("http") - async def add_strict_transport_security_header(request: Request, call_next): - response = await call_next(request) - response.headers[ - "Strict-Transport-Security" - ] = f"max-age={SECONDS_IN_TWO_YEARS}; includeSubDomains; preload" - return response - - @app.middleware("http") - async def add_x_frame_options_header(request: Request, call_next): - response = await call_next(request) - response.headers["X-Frame-Options"] = "sameorigin" - return response - - @app.middleware("http") - async def add_x_content_type_options_header(request: Request, call_next): - response = await call_next(request) - response.headers["X-Content-Type-Options"] = "nosniff" - return response - - # this has to registered after the first middleware but before sentry? - app.add_middleware(SuppressClientDisconnectNoResponseReturnedMiddleware) - - if settings.SENTRY_DSN: - sentry_sdk.init( - dsn=settings.SENTRY_DSN, - environment=settings.ENVIRONMENT, - traces_sample_rate=settings.SENTRY_TRACE_SAMPLE_RATE, - release=settings.RELEASE_TAG, - ) - app.add_middleware(SentryAsgiMiddleware) - - return app - - -# Supress "no response returned" error when client disconnects -# discussion and sample code found here -# https://github.com/encode/starlette/discussions/1527 -class SuppressClientDisconnectNoResponseReturnedMiddleware(BaseHTTPMiddleware): - async def dispatch( - self, request: Request, call_next: RequestResponseEndpoint - ) -> Response: - try: - response = await call_next(request) - except RuntimeError as e: - if await request.is_disconnected() and str(e) == "No response returned.": - logger.warning( - "Error 'No response returned' detected - but client already disconnected" - ) - return Response(status_code=HTTPStatus.NO_CONTENT) - else: - raise - return response +import logging +import sentry_sdk + +from datetime import timedelta + +from sqlalchemy.orm import Session +from fastapi import FastAPI, Request, status, Depends +from fastapi.encoders import jsonable_encoder +from fastapi.exception_handlers import http_exception_handler +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import HTMLResponse, JSONResponse, FileResponse +from fastapi.staticfiles import StaticFiles +from pydantic import ValidationError +from sentry_sdk.integrations.asgi import SentryAsgiMiddleware +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.responses import Response +from http import HTTPStatus + +from application.db.session import get_session +from application.core.templates import templates +from application.db.models import EntityOrm +from application.exceptions import DigitalLandValidationError +from application.routers import ( + entity, + dataset, + map_, + curie, + organisation, + fact, + guidance_, + about_, + tiles, + osMapOAuth, +) +from application.settings import get_settings + +logger = logging.getLogger(__name__) +settings = get_settings() + +SECONDS_IN_TWO_YEARS = timedelta(days=365 * 2).total_seconds() + +# Add markdown here +description = """ +## About this API +""" + +tags_metadata = [ + { + "name": "Search entity", + "description": "find entities by location, type or date", + }, + { + "name": "Get entity", + "description": "get entity by id", + }, + { + "name": "List datasets", + "description": "list all datasets", + }, + { + "name": "Get dataset", + "description": "get dataset by id", + }, +] + + +def create_app(): + app = FastAPI( + title="planning.data.gov.uk API", + description=description, + version="0.1.0", + contact={ + "name": "planning.data.gov.uk team", + "email": "digitalland@levellingup.gov.uk", + "url": "https://www.planning.data.gov.uk", + }, + license_info={ + "name": "MIT", + "url": "https://opensource.org/licenses/MIT", + }, + openapi_tags=tags_metadata, + docs_url=None, + redoc_url=None, + servers=[{"url": "https://www.planning.data.gov.uk"}], + ) + add_base_routes(app) + add_routers(app) + add_static(app) + app = add_middleware(app) + return app + + +def add_base_routes(app): + @app.get("/", response_class=HTMLResponse, include_in_schema=False) + def home(request: Request): + return templates.TemplateResponse( + "homepage.html", {"request": request, "opengraph_image": True} + ) + + @app.get("/health", response_class=JSONResponse, include_in_schema=False) + def health(session: Session = Depends(get_session)): + from sqlalchemy.sql import select + + try: + sql = select(EntityOrm.entity).limit(1) + result = session.execute(sql).fetchone() + status = { + "status": "OK", + "entities_present": "OK" if result is not None else "FAIL", + } + logger.info(f"healthcheck {status}") + return status + except Exception as e: + logger.exception(e) + raise e + + @app.get( + "/invalid-geometries", response_class=JSONResponse, include_in_schema=False + ) + def invalid_geometries(session: Session = Depends(get_session)): + from application.core.models import entity_factory + from sqlalchemy import func + from sqlalchemy import and_ + from sqlalchemy import not_ + + try: + query_args = [ + EntityOrm, + func.ST_IsValidReason(EntityOrm.geometry).label("invalid_reason"), + ] + query = session.query(*query_args) + query = query.filter( + and_( + EntityOrm.geometry.is_not(None), + not_(func.ST_IsValid(EntityOrm.geometry)), + ) + ) + entities = query.all() + return [ + { + "entity": entity_factory(e.EntityOrm), + "invalid_reason": e.invalid_reason, + } + for e in entities + ] + except Exception as e: + logger.exception(e) + return {"message": "There was an error checking for invalid geometries"} + + @app.get("/cookies", response_class=HTMLResponse, include_in_schema=False) + def cookies(request: Request): + return templates.TemplateResponse( + "pages/cookies.html", + {"request": request}, + ) + + @app.get( + "/accessibility-statement", response_class=HTMLResponse, include_in_schema=False + ) + def accessibility_statement(request: Request): + return templates.TemplateResponse( + "pages/accessibility-statement.html", + {"request": request}, + ) + + @app.get("/service-status", response_class=HTMLResponse, include_in_schema=False) + def service_status(request: Request): + return templates.TemplateResponse( + "pages/service-status.html", + {"request": request}, + ) + + @app.get("/docs", response_class=HTMLResponse, include_in_schema=False) + def docs(request: Request): + open_api_dict = app.openapi() + return templates.TemplateResponse( + "pages/docs.html", + { + "request": request, + "paths": open_api_dict["paths"], + "components": open_api_dict["components"], + }, + ) + + @app.get("/robots.txt", response_class=FileResponse, include_in_schema=False) + def robots(): + return FileResponse("static/robots.txt") + + @app.exception_handler(StarletteHTTPException) + async def custom_404_exception_handler( + request: Request, exc: StarletteHTTPException + ): + if exc.status_code == 404: + return templates.TemplateResponse( + "404.html", + {"request": request}, + status_code=exc.status_code, + ) + else: + # Just use FastAPI's built-in handler for other errors + return await http_exception_handler(request, exc) + + # FastAPI disapproves of handling ValidationErrors as they leak internal info to users + # Unfortunately, the errors raised by the validator bound to QueryFilters are not caught and + # reraised as RequestValidationError, so we handle that subset of ValidationErrors manually here + @app.exception_handler(ValidationError) + async def custom_validation_error_handler(request: Request, exc: ValidationError): + if all( + [ + isinstance(raw_error.exc, DigitalLandValidationError) + for raw_error in exc.raw_errors + ] + ): + try: + extension_path_param = request.path_params["extension"] + except KeyError: + extension_path_param = None + if extension_path_param in ["json", "geojson"]: + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content=jsonable_encoder({"detail": exc.errors()}), + ) + else: + return templates.TemplateResponse( + "404.html", + {"request": request}, + status_code=status.HTTP_404_NOT_FOUND, + ) + + else: + raise exc + + @app.exception_handler(RequestValidationError) + async def custom_request_validation_error_handler(request, exc): + try: + extension_path_param = request.path_params["extension"] + except KeyError: + extension_path_param = None + + if extension_path_param in ["json", "geojson"]: + return JSONResponse( + status_code=422, + content=jsonable_encoder({"detail": exc.errors()}), + ) + else: + return templates.TemplateResponse( + "404.html", {"request": request}, status_code=404 + ) + + # catch all handler - for any unhandled exceptions return 500 template + @app.exception_handler(Exception) + async def custom_catch_all_exception_handler(request: Request, exc: Exception): + return templates.TemplateResponse( + "500.html", {"request": request}, status_code=500 + ) + + +def add_routers(app): + app.include_router(entity.router, prefix="/entity") + app.include_router(dataset.router, prefix="/dataset") + app.include_router(curie.router, prefix="/curie") + app.include_router(curie.router, prefix="/prefix") + app.include_router(organisation.router, prefix="/organisation") + app.include_router(fact.router, prefix="/fact") + + # not added to /docs + app.include_router(osMapOAuth.router, prefix="/os", include_in_schema=False) + app.include_router(map_.router, prefix="/map", include_in_schema=False) + app.include_router(guidance_.router, prefix="/guidance", include_in_schema=False) + app.include_router(about_.router, prefix="/about", include_in_schema=False) + app.include_router(tiles.router, prefix="/tiles", include_in_schema=False) + + +def add_static(app): + app.mount( + "/static", + StaticFiles(directory="static"), + name="static", + ) + + +def add_middleware(app): + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=False, + allow_methods=["*"], + allow_headers=["*"], + ) + + @app.middleware("http") + async def add_strict_transport_security_header(request: Request, call_next): + response = await call_next(request) + response.headers["Strict-Transport-Security"] = ( + f"max-age={SECONDS_IN_TWO_YEARS}; includeSubDomains; preload" + ) + return response + + @app.middleware("http") + async def add_x_frame_options_header(request: Request, call_next): + response = await call_next(request) + response.headers["X-Frame-Options"] = "sameorigin" + return response + + @app.middleware("http") + async def add_x_content_type_options_header(request: Request, call_next): + response = await call_next(request) + response.headers["X-Content-Type-Options"] = "nosniff" + return response + + # this has to registered after the first middleware but before sentry? + app.add_middleware(SuppressClientDisconnectNoResponseReturnedMiddleware) + + if settings.SENTRY_DSN: + sentry_sdk.init( + dsn=settings.SENTRY_DSN, + environment=settings.ENVIRONMENT, + traces_sample_rate=settings.SENTRY_TRACE_SAMPLE_RATE, + release=settings.RELEASE_TAG, + ) + app.add_middleware(SentryAsgiMiddleware) + + return app + + +# Supress "no response returned" error when client disconnects +# discussion and sample code found here +# https://github.com/encode/starlette/discussions/1527 +class SuppressClientDisconnectNoResponseReturnedMiddleware(BaseHTTPMiddleware): + async def dispatch( + self, request: Request, call_next: RequestResponseEndpoint + ) -> Response: + try: + response = await call_next(request) + except RuntimeError as e: + if await request.is_disconnected() and str(e) == "No response returned.": + logger.warning( + "Error 'No response returned' detected - but client already disconnected" + ) + return Response(status_code=HTTPStatus.NO_CONTENT) + else: + raise + return response diff --git a/application/routers/tiles.py b/application/routers/tiles.py index 50325a02..0bc0b369 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -1,7 +1,7 @@ -from fastapi import APIRouter, HTTPException, Depends +from fastapi import APIRouter, HTTPException, Depends, Request from fastapi.responses import StreamingResponse from sqlalchemy.orm import Session -from sqlalchemy import func +from sqlalchemy import func, case, and_ from io import BytesIO from application.db.models import EntityOrm @@ -26,41 +26,39 @@ def tile_is_valid(tile): # Build the database query using SQLAlchemy ORM to match the direct SQL logic def build_db_query(tile, session: Session): + srid = 4326 # WGS 84 + + # Define the envelope, webmercator, and WGS84 transformations envelope = func.ST_TileEnvelope(tile["zoom"], tile["x"], tile["y"]) webmercator = envelope - srid = 4326 # WGS 84 wgs84 = func.ST_Transform(webmercator, srid) bounds = func.ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, srid) - geometries = ( - session.query( - EntityOrm.entity, - EntityOrm.name, - EntityOrm.reference, - func.ST_AsMVTGeom( - func.CASE( - [ - ( - func.ST_Covers(bounds, EntityOrm.geometry), - func.ST_Transform(EntityOrm.geometry, srid), - ) - ], - else_=func.ST_Transform( - func.ST_Intersection(bounds, EntityOrm.geometry), srid - ), - ), - wgs84, - ), - ) + # Define the CASE expression for geometry transformation + geometry_case = case( + [ + ( + func.ST_Covers(bounds, EntityOrm.geometry), + func.ST_Transform(EntityOrm.geometry, srid), + ) + ], + else_=func.ST_Transform(func.ST_Intersection(bounds, EntityOrm.geometry), srid), + ) + + # Geometry processing for MVT + query = ( + session.query(func.ST_AsMVTGeom(geometry_case, wgs84).label("geom")) .filter( - EntityOrm.geometry.ST_Intersects(wgs84), - EntityOrm.dataset == tile["dataset"], + and_( + func.ST_Intersects(EntityOrm.geometry, wgs84), + EntityOrm.dataset == tile["dataset"], + ) ) .subquery() ) - # Build vector tile - tile_data = session.query(func.ST_AsMVT(geometries, tile["dataset"])).scalar() + # Generate MVT from a single-column subquery + tile_data = session.query(func.ST_AsMVT(query.c.geom, tile["dataset"])).scalar() return tile_data @@ -70,8 +68,9 @@ def build_db_query(tile, session: Session): # ============================================================ -@router.get("/tiles/{dataset}/{z}/{x}/{y}.{fmt}") +@router.get("/{dataset}/{z}/{x}/{y}.vector.{fmt}") async def read_tiles_from_postgres( + request: Request, dataset: str, z: int, x: int, @@ -79,6 +78,7 @@ async def read_tiles_from_postgres( fmt: str, session: Session = Depends(get_session), ): + print("Hello", {dataset}) tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} if not tile_is_valid(tile): raise HTTPException(status_code=400, detail=f"Invalid tile path: {tile}") diff --git a/application/templates/components/map/macro.jinja b/application/templates/components/map/macro.jinja index fcf3a2fb..4521f82a 100644 --- a/application/templates/components/map/macro.jinja +++ b/application/templates/components/map/macro.jinja @@ -53,14 +53,14 @@ params = { ...params, baseTileStyleFilePath: "/static/javascripts/base-tile.json", - vectorSource: "{{ params.DATASETTE_TILES_URL }}/-/tiles/dataset_tiles/{z}/{x}/{y}.vector.pbf", - datasetVectorUrl: "{{ params.DATASETTE_TILES_URL }}/-/tiles/", + vectorSource: "{{ params.DATASETTE_TILES_URL }}/tiles/{z}/{x}/{y}.vector.pbf", + datasetVectorUrl: "{{ params.DATASETTE_TILES_URL }}/tiles/", datasets: {{layers|tojson}}.map(d => d.dataset), vectorTileSources: {{layers|tojson}}.map(d => { d.paint_options = d.paint_options || {}; return { name: d.dataset, - vectorSource: "{{ params.DATASETTE_TILES_URL }}/-/tiles/" + d.dataset + "/{z}/{x}/{y}.vector.pbf", + vectorSource: "{{ params.DATASETTE_TILES_URL }}/tiles/" + d.dataset + "/{z}/{x}/{y}.vector.pbf", dataType: d.paint_options.type, styleProps: { colour: d.paint_options.colour, diff --git a/assets/javascripts/MapController.js b/assets/javascripts/MapController.js index 00f00b4f..998a1254 100644 --- a/assets/javascripts/MapController.js +++ b/assets/javascripts/MapController.js @@ -1,651 +1,651 @@ -import BrandImageControl from "./BrandImageControl.js"; -import CopyrightControl from "./CopyrightControl.js"; -import LayerControls from "./LayerControls.js"; -import TiltControl from "./TiltControl.js"; -import { capitalizeFirstLetter, preventScroll } from "./utils.js"; -import { getApiToken, getFreshApiToken } from "./osApiToken.js"; -import { defaultPaintOptions } from "./defaultPaintOptions.js"; - -export default class MapController { - constructor(params) { - // set the params applying default values where none were provided - this.setParams(params); - - // create an array to store the geojson layers - this.geojsonLayers = []; - - // create the maplibre map - this.createMap(); - } - - setParams(params) { - params = params || {}; - this.mapId = params.mapId || "mapid"; - this.mapContainerSelector = - params.mapContainerSelector || ".dl-map__wrapper"; - this.vectorTileSources = params.vectorTileSources || []; - this.datasetVectorUrl = - params.datasetVectorUrl || "http://"; - this.apiKey = params.apiKey || null; - this.datasets = params.datasets || null; - this.minMapZoom = params.minMapZoom || 5; - this.maxMapZoom = params.maxMapZoom || 15; - this.baseURL = params.baseURL || "https://digital-land.github.io"; - this.baseTileStyleFilePath = - params.baseTileStyleFilePath || "/static/javascripts/base-tile.json"; - this.popupWidth = params.popupWidth || "260px"; - this.popupMaxListLength = params.popupMaxListLength || 10; - this.LayerControlOptions = params.LayerControlOptions || { enabled: false }; - this.ZoomControlsOptions = params.ZoomControlsOptions || { enabled: false }; - this.FullscreenControl = params.FullscreenControl || { enabled: false }; - this.geojsons = params.geojsons || []; - this.images = params.images || [ - { - src: "/static/images/location-pointer-sdf-256.png", - name: "custom-marker-256", - size: 256, - }, - ]; - this.paint_options = params.paint_options || null; - this.customStyleJson = "/static/javascripts/base-tile.json"; - this.customStyleLayersToBringToFront = ["OS/Names/National/Country"]; - this.useOAuth2 = params.useOAuth2 || false; - this.layers = params.layers || []; - this.featuresHoveringOver = 0; - } - - getViewFromUrl() { - const urlObj = new URL(document.location); - const hash = urlObj.hash; - if (hash) { - const [lat, lng, zoom] = hash.substring(1).split(","); - return { - centre: [parseFloat(lng), parseFloat(lat)], - zoom: parseFloat(zoom), - }; - } - return { centre: undefined, zoom: undefined }; - } - - async createMap() { - // Define the custom JSON style. - // More styles can be found at https://github.com/OrdnanceSurvey/OS-Vector-Tile-API-Stylesheets. - - await getFreshApiToken(); - - const viewFromUrl = this.getViewFromUrl(); - - var map = new maplibregl.Map({ - container: this.mapId, - minZoom: 5.5, - maxZoom: 18, - style: this.customStyleJson, - maxBounds: [ - [-15, 49], - [13, 57], - ], - center: viewFromUrl.centre || [-1, 52.9], - zoom: viewFromUrl.zoom || 5.5, - transformRequest: (url, resourceType) => { - if (url.startsWith(this.datasetVectorUrl)) { - // Check if the request URL is for your tile server - const newUrl = new URL(url); - if (this.useOAuth2) { - return { - url: newUrl.toString(), - headers: { Authorization: "Bearer " + getApiToken() }, - }; - } else { - newUrl.searchParams.append("key", this.apiKey); - return { url: newUrl.toString() }; - } - } - return { url }; - }, - }); - - map.getCanvas().ariaLabel = `${this.mapId}`; - this.map = map; - - // once the maplibre map has loaded call the setup function - var boundSetup = this.setup.bind(this); - this.map.on("load", boundSetup); - } - - async setup() { - console.log("setup"); - try { - await this.loadImages(this.images); - } catch (e) { - console.log("error loading images: " + e); - } - console.log("past load images"); - this.availableLayers = this.addVectorTileSources(this.vectorTileSources); - this.geojsonLayers = this.addGeojsonSources(this.geojsons); - if (this.geojsonLayers.length == 1) { - this.flyTo(this.geojsons[0]); - } - this.addControls(); - this.addClickHandlers(); - this.overwriteWheelEventsForControls(); - - const handleMapMove = () => { - const center = this.map.getCenter(); - const zoom = this.map.getZoom(); - const urlObj = new URL(document.location); - const newURL = - urlObj.origin + - urlObj.pathname + - urlObj.search + - `#${center.lat},${center.lng},${zoom}z`; - window.history.replaceState({}, "", newURL); - }; - this.obscureScotland(); - this.obscureWales(); - this.addNeighbours(); - this.map.on("moveend", handleMapMove); - } - - loadImages(imageSrc = []) { - console.log("loading images" + imageSrc.length + " images"); - return new Promise((resolve, reject) => { - const promiseArray = imageSrc.map(({ src, name }) => { - return new Promise((resolve, reject) => { - this.map.loadImage(src, (error, image) => { - if (error) { - console.log("error adding image: " + error); - reject(error); - } - console.log("added image"); - this.map.addImage(name, image, { sdf: true }); - resolve(); - }); - }); - }); - Promise.all(promiseArray) - .then(() => { - console.log("resolved"); - resolve(); - }) - .catch((error) => { - console.log("rejected"); - reject(error); - }); - }); - } - - addVectorTileSources(vectorTileSources = []) { - let availableLayers = {}; - // add vector tile sources to map - vectorTileSources.forEach((source) => { - let layers = this.addVectorTileSource(source); - availableLayers[source.name] = layers; - }); - return availableLayers; - } - - obscureWales() { - this.obscure("Wales_simplified", "#FFFFFF", 0.6); - } - - obscureScotland() { - this.obscure("Scotland_simplified"); - } - - addNeighbours() { - this.obscure("UK_neighbours", "#FFFFFF", 0.9); - } - - obscure(name, colour = "#FFFFFF", opacity = 0.8) { - this.map.addSource(name, { - type: "geojson", - data: `/static/javascripts/geojsons/${name}.json`, - buffer: 0, - }); - const layerId = `${name}_Layer`; - this.map.addLayer({ - id: layerId, - type: "fill", - source: name, - layout: {}, - paint: { - "fill-color": colour, - "fill-opacity": opacity, - }, - }); - this.map.moveLayer(layerId, "OS/Names/National/Country"); - } - - addGeojsonSources(geojsons = []) { - // add geojsons sources to map - const addedLayers = []; - geojsons.forEach((geojson) => { - if (geojson.data.type == "Point") - addedLayers.push(this.addPoint(geojson, this.images[0])); - else if (["Polygon", "MultiPolygon"].includes(geojson.data.type)) - addedLayers.push(this.addPolygon(geojson)); - else throw new Error("Unsupported geometry type"); - }); - return addedLayers; - } - - addControls() { - this.map.addControl( - new maplibregl.ScaleControl({ - container: document.getElementById(this.mapId), - }), - "bottom-left" - ); - - if (this.FullscreenControl.enabled) { - this.map.addControl( - new maplibregl.FullscreenControl({ - container: document.getElementById(this.mapId), - }), - "top-left" - ); - } - this.map.addControl(new TiltControl(), "top-left"); - this.map.addControl( - new maplibregl.NavigationControl({ - container: document.getElementById(this.mapId), - }), - "top-left" - ); - - this.map.addControl(new CopyrightControl(), "bottom-right"); - - if (this.LayerControlOptions.enabled) { - this.layerControlsComponent = new LayerControls( - this, - this.sourceName, - this.layers, - this.availableLayers, - this.LayerControlOptions - ); - this.map.addControl(this.layerControlsComponent, "top-right"); - } - } - - overwriteWheelEventsForControls() { - const mapEl = document.getElementById(this.mapId); - const mapControlsArray = mapEl.querySelectorAll( - ".maplibregl-control-container" - ); - mapControlsArray.forEach((mapControls) => - mapControls.addEventListener( - "wheel", - preventScroll([".dl-map__side-panel__content"]), - { passive: false } - ) - ); - } - - addClickHandlers() { - if (this.layerControlsComponent) { - this.map.on("click", this.clickHandler.bind(this)); - } - } - - flyTo(geometry) { - if (geometry.data.type == "Point") { - this.map.flyTo({ - center: geometry.data.coordinates, - essential: true, - animate: false, - zoom: 15, - }); - } else { - var bbox = turf.extent(geometry.data); - this.map.fitBounds(bbox, { padding: 20, animate: false }); - } - } - - addLayer({ - sourceName, - layerType, - paintOptions = {}, - layoutOptions = {}, - sourceLayer = "", - additionalOptions = {}, - }) { - const layerName = `${sourceName}-${layerType}`; - this.map.addLayer({ - id: layerName, - type: layerType, - source: sourceName, - "source-layer": sourceLayer, - paint: paintOptions, - layout: layoutOptions, - ...additionalOptions, - }); - - if (["fill", "fill-extrusion", "circle"].includes(layerType)) { - this.map.on("mouseover", layerName, () => { - this.map.getCanvas().style.cursor = "pointer"; - this.featuresHoveringOver++; - }); - this.map.on("mouseout", layerName, () => { - this.featuresHoveringOver--; - if (this.featuresHoveringOver == 0) - this.map.getCanvas().style.cursor = ""; - }); - } - - return layerName; - } - - addPolygon(geometry) { - this.map.addSource(geometry.name, { - type: "geojson", - data: { - type: "Feature", - geometry: geometry.data, - properties: { - entity: geometry.entity, - name: geometry.name, - }, - }, - }); - - let colour = "blue"; - if (this.paint_options) colour = this.paint_options.colour; - - let layer = this.addLayer({ - sourceName: geometry.name, - layerType: "fill-extrusion", - paintOptions: { - "fill-extrusion-color": colour, - "fill-extrusion-opacity": 0.5, - "fill-extrusion-height": 1, - "fill-extrusion-base": 0, - }, - }); - - this.moveLayerBehindBuildings(layer); - - return layer; - } - - moveLayerBehindBuildings( - layer, - buildingsLayer = "OS/TopographicArea_1/Building/1_3D" - ) { - try { - this.map.moveLayer(layer, buildingsLayer); - } catch (e) { - console.error(`Could not move layer behind ${buildingsLayer}: `, e); - } - } - - addPoint(geometry, image = undefined) { - this.map.addSource(geometry.name, { - type: "geojson", - data: { - type: "Feature", - geometry: geometry.data, - properties: { - entity: geometry.entity, - name: geometry.name, - }, - }, - }); - - let iconColor = "blue"; - if (this.paint_options) iconColor = this.paint_options.colour; - - let layerName; - // if an image is provided use that otherwise use a circle - if (image) { - if (!this.map.hasImage(image.name)) { - throw new Error( - "Image not loaded, imageName: " + image.name + " not found" - ); - } - layerName = this.addLayer({ - sourceName: geometry.name, - layerType: "symbol", - paintOptions: { - "icon-color": iconColor, - "icon-opacity": 1, - }, - layoutOptions: { - "icon-image": image.name, - "icon-size": (256 / image.size) * 0.15, - "icon-anchor": "bottom", - // get the year from the source's "year" property - "text-field": ["get", "year"], - "text-font": ["Open Sans Semibold", "Arial Unicode MS Bold"], - "text-offset": [0, 1.25], - "text-anchor": "top", - }, - }); - } else { - layerName = this.addLayer({ - sourceName: geometry.name, - layerType: "circle", - paintOptions: { - "circle-color": iconColor, - "circle-radius": defaultPaintOptions["circle-radius"], - }, - }); - } - return layerName; - } - - addVectorTileSource(source) { - // add source - this.map.addSource(`${source.name}-source`, { - type: "vector", - tiles: [source.vectorSource], - minzoom: this.minMapZoom, - maxzoom: this.maxMapZoom, - }); - - // add layer - let layers; - if (source.dataType === "point") { - let layerName = this.addLayer({ - sourceName: `${source.name}-source`, - layerType: "circle", - paintOptions: { - "circle-color": - source.styleProps.colour || defaultPaintOptions["fill-color"], - "circle-opacity": - source.styleProps.opacity || defaultPaintOptions["fill-opacity"], - "circle-stroke-color": - source.styleProps.colour || defaultPaintOptions["fill-color"], - "circle-radius": defaultPaintOptions["circle-radius"], - }, - sourceLayer: `${source.name}`, - }); - - layers = [layerName]; - } else { - // create fill layer - let fillLayerName = this.addLayer({ - sourceName: `${source.name}-source`, - layerType: "fill-extrusion", - paintOptions: { - "fill-extrusion-color": - source.styleProps.colour || defaultPaintOptions["fill-color"], - "fill-extrusion-height": 1, - "fill-extrusion-base": 0, - "fill-extrusion-opacity": - parseFloat(source.styleProps.opacity) || - defaultPaintOptions["fill-opacity"], - }, - sourceLayer: `${source.name}`, - }); - - this.moveLayerBehindBuildings(fillLayerName); - - // create line layer - let lineLayerName = this.addLayer({ - sourceName: `${source.name}-source`, - layerType: "line", - paintOptions: { - "line-color": - source.styleProps.colour || defaultPaintOptions["fill-color"], - "line-width": - source.styleProps.weight || defaultPaintOptions["weight"], - }, - sourceLayer: `${source.name}`, - }); - - // create point layer for geometries - let pointLayerName = this.addLayer({ - sourceName: `${source.name}-source`, - layerType: "circle", - paintOptions: { - "circle-color": - source.styleProps.colour || defaultPaintOptions["fill-color"], - "circle-opacity": - source.styleProps.opacity || defaultPaintOptions["fill-opacity"], - "circle-stroke-color": - source.styleProps.colour || defaultPaintOptions["fill-color"], - "circle-radius": defaultPaintOptions["circle-radius"], - }, - sourceLayer: `${source.name}`, - additionalOptions: { - filter: ["==", ["geometry-type"], "Point"], - }, - }); - layers = [fillLayerName, lineLayerName, pointLayerName]; - } - return layers; - } - - clickHandler(e) { - var map = this.map; - var bbox = [ - [e.point.x - 5, e.point.y - 5], - [e.point.x + 5, e.point.y + 5], - ]; - - let clickableLayers = - this.layerControlsComponent.getClickableLayers() || []; - - var features = map.queryRenderedFeatures(bbox, { - layers: clickableLayers, - }); - var coordinates = e.lngLat; - - if (features.length) { - // no need to show popup if not clicking on feature - var popupDomElement = this.createFeaturesPopup( - this.removeDuplicates(features) - ); - var popup = new maplibregl.Popup({ - maxWidth: this.popupWidth, - }) - .setLngLat(coordinates) - .setDOMContent(popupDomElement) - .addTo(map); - popup.getElement().onwheel = preventScroll([".app-popup-list"]); - } - } - - // map.queryRenderedFeatures() can return duplicate features so we need to remove them - removeDuplicates(features) { - var uniqueEntities = []; - - return features.filter(function (feature) { - if (uniqueEntities.indexOf(feature.properties.entity) === -1) { - uniqueEntities.push(feature.properties.entity); - return true; - } - - return false; - }); - } - - createFeaturesPopup(features) { - const wrapper = document.createElement("div"); - wrapper.classList.add("app-popup"); - - const featureOrFeatures = features.length > 1 ? "features" : "feature"; - const heading = document.createElement("h3"); - heading.classList.add("app-popup-heading"); - heading.textContent = `${features.length} ${featureOrFeatures} selected`; - wrapper.appendChild(heading); - - if (features.length > this.popupMaxListLength) { - const tooMany = document.createElement("p"); - tooMany.classList.add("govuk-body-s"); - tooMany.textContent = `You clicked on ${features.length} features.`; - const tooMany2 = document.createElement("p"); - tooMany2.classList.add("govuk-body-s"); - tooMany2.textContent = - "Zoom in or turn off layers to narrow down your choice."; - wrapper.appendChild(tooMany); - wrapper.appendChild(tooMany2); - return wrapper; - } - - const list = document.createElement("ul"); - list.classList.add("app-popup-list"); - features.forEach((feature) => { - const featureType = capitalizeFirstLetter( - feature.sourceLayer || feature.source - ).replaceAll("-", " "); - const fillColour = this.getFillColour(feature); - - const featureName = - feature.properties.name || feature.properties.reference || "Not Named"; - const item = document.createElement("li"); - item.classList.add("app-popup-item"); - item.style.borderLeft = `5px solid ${fillColour}`; - - const secondaryText = document.createElement("p"); - secondaryText.classList.add( - "app-u-secondary-text", - "govuk-!-margin-bottom-0", - "govuk-!-margin-top-0" - ); - secondaryText.textContent = featureType; - item.appendChild(secondaryText); - - const link = document.createElement("a"); - link.classList.add("govuk-link"); - link.href = `/entity/${feature.properties.entity}`; - link.textContent = featureName; - const smallText = document.createElement("p"); - smallText.classList.add( - "dl-small-text", - "govuk-!-margin-top-0", - "govuk-!-margin-bottom-0" - ); - smallText.appendChild(link); - item.appendChild(smallText); - - list.appendChild(item); - }); - - wrapper.appendChild(list); - return wrapper; - } - - getFillColour(feature) { - if (feature.layer.type === "symbol") - return this.map.getLayer(feature.layer.id).getPaintProperty("icon-color"); - else if (feature.layer.type === "fill") - return this.map.getLayer(feature.layer.id).getPaintProperty("fill-color"); - else if (feature.layer.type === "fill-extrusion") - return this.map - .getLayer(feature.layer.id) - .getPaintProperty("fill-extrusion-color"); - else if (feature.layer.type === "circle") - return this.map - .getLayer(feature.layer.id) - .getPaintProperty("circle-color"); - else - throw new Error( - "could not get fill colour for feature of type " + feature.layer.type - ); - } - - setLayerVisibility(layerName, visibility) { - this.map.setLayoutProperty(layerName, "visibility", visibility); - } -} +import BrandImageControl from "./BrandImageControl.js"; +import CopyrightControl from "./CopyrightControl.js"; +import LayerControls from "./LayerControls.js"; +import TiltControl from "./TiltControl.js"; +import { capitalizeFirstLetter, preventScroll } from "./utils.js"; +import { getApiToken, getFreshApiToken } from "./osApiToken.js"; +import { defaultPaintOptions } from "./defaultPaintOptions.js"; + +export default class MapController { + constructor(params) { + // set the params applying default values where none were provided + this.setParams(params); + + // create an array to store the geojson layers + this.geojsonLayers = []; + + // create the maplibre map + this.createMap(); + } + + setParams(params) { + params = params || {}; + this.mapId = params.mapId || "mapid"; + this.mapContainerSelector = + params.mapContainerSelector || ".dl-map__wrapper"; + this.vectorTileSources = params.vectorTileSources || []; + this.datasetVectorUrl = + params.datasetVectorUrl || "http://"; + this.apiKey = params.apiKey || null; + this.datasets = params.datasets || null; + this.minMapZoom = params.minMapZoom || 5; + this.maxMapZoom = params.maxMapZoom || 15; + this.baseURL = params.baseURL || "https://digital-land.github.io"; + this.baseTileStyleFilePath = + params.baseTileStyleFilePath || "/static/javascripts/base-tile.json"; + this.popupWidth = params.popupWidth || "260px"; + this.popupMaxListLength = params.popupMaxListLength || 10; + this.LayerControlOptions = params.LayerControlOptions || { enabled: false }; + this.ZoomControlsOptions = params.ZoomControlsOptions || { enabled: false }; + this.FullscreenControl = params.FullscreenControl || { enabled: false }; + this.geojsons = params.geojsons || []; + this.images = params.images || [ + { + src: "/static/images/location-pointer-sdf-256.png", + name: "custom-marker-256", + size: 256, + }, + ]; + this.paint_options = params.paint_options || null; + this.customStyleJson = "/static/javascripts/base-tile.json"; + this.customStyleLayersToBringToFront = ["OS/Names/National/Country"]; + this.useOAuth2 = params.useOAuth2 || false; + this.layers = params.layers || []; + this.featuresHoveringOver = 0; + } + + getViewFromUrl() { + const urlObj = new URL(document.location); + const hash = urlObj.hash; + if (hash) { + const [lat, lng, zoom] = hash.substring(1).split(","); + return { + centre: [parseFloat(lng), parseFloat(lat)], + zoom: parseFloat(zoom), + }; + } + return { centre: undefined, zoom: undefined }; + } + + async createMap() { + // Define the custom JSON style. + // More styles can be found at https://github.com/OrdnanceSurvey/OS-Vector-Tile-API-Stylesheets. + + await getFreshApiToken(); + + const viewFromUrl = this.getViewFromUrl(); + + var map = new maplibregl.Map({ + container: this.mapId, + minZoom: 5.5, + maxZoom: 18, + style: this.customStyleJson, + maxBounds: [ + [-15, 49], + [13, 57], + ], + center: viewFromUrl.centre || [-1, 52.9], + zoom: viewFromUrl.zoom || 5.5, + transformRequest: (url, resourceType) => { + if (url.startsWith(this.datasetVectorUrl)) { + // Check if the request URL is for your tile server + const newUrl = new URL(url); + if (this.useOAuth2) { + return { + url: newUrl.toString(), + headers: { Authorization: "Bearer " + getApiToken() }, + }; + } else { + newUrl.searchParams.append("key", this.apiKey); + return { url: newUrl.toString() }; + } + } + return { url }; + }, + }); + + map.getCanvas().ariaLabel = `${this.mapId}`; + this.map = map; + + // once the maplibre map has loaded call the setup function + var boundSetup = this.setup.bind(this); + this.map.on("load", boundSetup); + } + + async setup() { + console.log("setup"); + try { + await this.loadImages(this.images); + } catch (e) { + console.log("error loading images: " + e); + } + console.log("past load images"); + this.availableLayers = this.addVectorTileSources(this.vectorTileSources); + this.geojsonLayers = this.addGeojsonSources(this.geojsons); + if (this.geojsonLayers.length == 1) { + this.flyTo(this.geojsons[0]); + } + this.addControls(); + this.addClickHandlers(); + this.overwriteWheelEventsForControls(); + + const handleMapMove = () => { + const center = this.map.getCenter(); + const zoom = this.map.getZoom(); + const urlObj = new URL(document.location); + const newURL = + urlObj.origin + + urlObj.pathname + + urlObj.search + + `#${center.lat},${center.lng},${zoom}z`; + window.history.replaceState({}, "", newURL); + }; + this.obscureScotland(); + this.obscureWales(); + this.addNeighbours(); + this.map.on("moveend", handleMapMove); + } + + loadImages(imageSrc = []) { + console.log("loading images" + imageSrc.length + " images"); + return new Promise((resolve, reject) => { + const promiseArray = imageSrc.map(({ src, name }) => { + return new Promise((resolve, reject) => { + this.map.loadImage(src, (error, image) => { + if (error) { + console.log("error adding image: " + error); + reject(error); + } + console.log("added image"); + this.map.addImage(name, image, { sdf: true }); + resolve(); + }); + }); + }); + Promise.all(promiseArray) + .then(() => { + console.log("resolved"); + resolve(); + }) + .catch((error) => { + console.log("rejected"); + reject(error); + }); + }); + } + + addVectorTileSources(vectorTileSources = []) { + let availableLayers = {}; + // add vector tile sources to map + vectorTileSources.forEach((source) => { + let layers = this.addVectorTileSource(source); + availableLayers[source.name] = layers; + }); + return availableLayers; + } + + obscureWales() { + this.obscure("Wales_simplified", "#FFFFFF", 0.6); + } + + obscureScotland() { + this.obscure("Scotland_simplified"); + } + + addNeighbours() { + this.obscure("UK_neighbours", "#FFFFFF", 0.9); + } + + obscure(name, colour = "#FFFFFF", opacity = 0.8) { + this.map.addSource(name, { + type: "geojson", + data: `/static/javascripts/geojsons/${name}.json`, + buffer: 0, + }); + const layerId = `${name}_Layer`; + this.map.addLayer({ + id: layerId, + type: "fill", + source: name, + layout: {}, + paint: { + "fill-color": colour, + "fill-opacity": opacity, + }, + }); + this.map.moveLayer(layerId, "OS/Names/National/Country"); + } + + addGeojsonSources(geojsons = []) { + // add geojsons sources to map + const addedLayers = []; + geojsons.forEach((geojson) => { + if (geojson.data.type == "Point") + addedLayers.push(this.addPoint(geojson, this.images[0])); + else if (["Polygon", "MultiPolygon"].includes(geojson.data.type)) + addedLayers.push(this.addPolygon(geojson)); + else throw new Error("Unsupported geometry type"); + }); + return addedLayers; + } + + addControls() { + this.map.addControl( + new maplibregl.ScaleControl({ + container: document.getElementById(this.mapId), + }), + "bottom-left" + ); + + if (this.FullscreenControl.enabled) { + this.map.addControl( + new maplibregl.FullscreenControl({ + container: document.getElementById(this.mapId), + }), + "top-left" + ); + } + this.map.addControl(new TiltControl(), "top-left"); + this.map.addControl( + new maplibregl.NavigationControl({ + container: document.getElementById(this.mapId), + }), + "top-left" + ); + + this.map.addControl(new CopyrightControl(), "bottom-right"); + + if (this.LayerControlOptions.enabled) { + this.layerControlsComponent = new LayerControls( + this, + this.sourceName, + this.layers, + this.availableLayers, + this.LayerControlOptions + ); + this.map.addControl(this.layerControlsComponent, "top-right"); + } + } + + overwriteWheelEventsForControls() { + const mapEl = document.getElementById(this.mapId); + const mapControlsArray = mapEl.querySelectorAll( + ".maplibregl-control-container" + ); + mapControlsArray.forEach((mapControls) => + mapControls.addEventListener( + "wheel", + preventScroll([".dl-map__side-panel__content"]), + { passive: false } + ) + ); + } + + addClickHandlers() { + if (this.layerControlsComponent) { + this.map.on("click", this.clickHandler.bind(this)); + } + } + + flyTo(geometry) { + if (geometry.data.type == "Point") { + this.map.flyTo({ + center: geometry.data.coordinates, + essential: true, + animate: false, + zoom: 15, + }); + } else { + var bbox = turf.extent(geometry.data); + this.map.fitBounds(bbox, { padding: 20, animate: false }); + } + } + + addLayer({ + sourceName, + layerType, + paintOptions = {}, + layoutOptions = {}, + sourceLayer = "", + additionalOptions = {}, + }) { + const layerName = `${sourceName}-${layerType}`; + this.map.addLayer({ + id: layerName, + type: layerType, + source: sourceName, + "source-layer": sourceLayer, + paint: paintOptions, + layout: layoutOptions, + ...additionalOptions, + }); + + if (["fill", "fill-extrusion", "circle"].includes(layerType)) { + this.map.on("mouseover", layerName, () => { + this.map.getCanvas().style.cursor = "pointer"; + this.featuresHoveringOver++; + }); + this.map.on("mouseout", layerName, () => { + this.featuresHoveringOver--; + if (this.featuresHoveringOver == 0) + this.map.getCanvas().style.cursor = ""; + }); + } + + return layerName; + } + + addPolygon(geometry) { + this.map.addSource(geometry.name, { + type: "geojson", + data: { + type: "Feature", + geometry: geometry.data, + properties: { + entity: geometry.entity, + name: geometry.name, + }, + }, + }); + + let colour = "blue"; + if (this.paint_options) colour = this.paint_options.colour; + + let layer = this.addLayer({ + sourceName: geometry.name, + layerType: "fill-extrusion", + paintOptions: { + "fill-extrusion-color": colour, + "fill-extrusion-opacity": 0.5, + "fill-extrusion-height": 1, + "fill-extrusion-base": 0, + }, + }); + + this.moveLayerBehindBuildings(layer); + + return layer; + } + + moveLayerBehindBuildings( + layer, + buildingsLayer = "OS/TopographicArea_1/Building/1_3D" + ) { + try { + this.map.moveLayer(layer, buildingsLayer); + } catch (e) { + console.error(`Could not move layer behind ${buildingsLayer}: `, e); + } + } + + addPoint(geometry, image = undefined) { + this.map.addSource(geometry.name, { + type: "geojson", + data: { + type: "Feature", + geometry: geometry.data, + properties: { + entity: geometry.entity, + name: geometry.name, + }, + }, + }); + + let iconColor = "blue"; + if (this.paint_options) iconColor = this.paint_options.colour; + + let layerName; + // if an image is provided use that otherwise use a circle + if (image) { + if (!this.map.hasImage(image.name)) { + throw new Error( + "Image not loaded, imageName: " + image.name + " not found" + ); + } + layerName = this.addLayer({ + sourceName: geometry.name, + layerType: "symbol", + paintOptions: { + "icon-color": iconColor, + "icon-opacity": 1, + }, + layoutOptions: { + "icon-image": image.name, + "icon-size": (256 / image.size) * 0.15, + "icon-anchor": "bottom", + // get the year from the source's "year" property + "text-field": ["get", "year"], + "text-font": ["Open Sans Semibold", "Arial Unicode MS Bold"], + "text-offset": [0, 1.25], + "text-anchor": "top", + }, + }); + } else { + layerName = this.addLayer({ + sourceName: geometry.name, + layerType: "circle", + paintOptions: { + "circle-color": iconColor, + "circle-radius": defaultPaintOptions["circle-radius"], + }, + }); + } + return layerName; + } + + addVectorTileSource(source) { + // add source + this.map.addSource(`${source.name}-source`, { + type: "vector", + tiles: [source.vectorSource], + minzoom: this.minMapZoom, + maxzoom: this.maxMapZoom, + }); + + // add layer + let layers; + if (source.dataType === "point") { + let layerName = this.addLayer({ + sourceName: `${source.name}-source`, + layerType: "circle", + paintOptions: { + "circle-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "circle-opacity": + source.styleProps.opacity || defaultPaintOptions["fill-opacity"], + "circle-stroke-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "circle-radius": defaultPaintOptions["circle-radius"], + }, + sourceLayer: `${source.name}`, + }); + + layers = [layerName]; + } else { + // create fill layer + let fillLayerName = this.addLayer({ + sourceName: `${source.name}-source`, + layerType: "fill-extrusion", + paintOptions: { + "fill-extrusion-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "fill-extrusion-height": 1, + "fill-extrusion-base": 0, + "fill-extrusion-opacity": + parseFloat(source.styleProps.opacity) || + defaultPaintOptions["fill-opacity"], + }, + sourceLayer: `${source.name}`, + }); + + this.moveLayerBehindBuildings(fillLayerName); + + // create line layer + let lineLayerName = this.addLayer({ + sourceName: `${source.name}-source`, + layerType: "line", + paintOptions: { + "line-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "line-width": + source.styleProps.weight || defaultPaintOptions["weight"], + }, + sourceLayer: `${source.name}`, + }); + + // create point layer for geometries + let pointLayerName = this.addLayer({ + sourceName: `${source.name}-source`, + layerType: "circle", + paintOptions: { + "circle-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "circle-opacity": + source.styleProps.opacity || defaultPaintOptions["fill-opacity"], + "circle-stroke-color": + source.styleProps.colour || defaultPaintOptions["fill-color"], + "circle-radius": defaultPaintOptions["circle-radius"], + }, + sourceLayer: `${source.name}`, + additionalOptions: { + filter: ["==", ["geometry-type"], "Point"], + }, + }); + layers = [fillLayerName, lineLayerName, pointLayerName]; + } + return layers; + } + + clickHandler(e) { + var map = this.map; + var bbox = [ + [e.point.x - 5, e.point.y - 5], + [e.point.x + 5, e.point.y + 5], + ]; + + let clickableLayers = + this.layerControlsComponent.getClickableLayers() || []; + + var features = map.queryRenderedFeatures(bbox, { + layers: clickableLayers, + }); + var coordinates = e.lngLat; + + if (features.length) { + // no need to show popup if not clicking on feature + var popupDomElement = this.createFeaturesPopup( + this.removeDuplicates(features) + ); + var popup = new maplibregl.Popup({ + maxWidth: this.popupWidth, + }) + .setLngLat(coordinates) + .setDOMContent(popupDomElement) + .addTo(map); + popup.getElement().onwheel = preventScroll([".app-popup-list"]); + } + } + + // map.queryRenderedFeatures() can return duplicate features so we need to remove them + removeDuplicates(features) { + var uniqueEntities = []; + + return features.filter(function (feature) { + if (uniqueEntities.indexOf(feature.properties.entity) === -1) { + uniqueEntities.push(feature.properties.entity); + return true; + } + + return false; + }); + } + + createFeaturesPopup(features) { + const wrapper = document.createElement("div"); + wrapper.classList.add("app-popup"); + + const featureOrFeatures = features.length > 1 ? "features" : "feature"; + const heading = document.createElement("h3"); + heading.classList.add("app-popup-heading"); + heading.textContent = `${features.length} ${featureOrFeatures} selected`; + wrapper.appendChild(heading); + + if (features.length > this.popupMaxListLength) { + const tooMany = document.createElement("p"); + tooMany.classList.add("govuk-body-s"); + tooMany.textContent = `You clicked on ${features.length} features.`; + const tooMany2 = document.createElement("p"); + tooMany2.classList.add("govuk-body-s"); + tooMany2.textContent = + "Zoom in or turn off layers to narrow down your choice."; + wrapper.appendChild(tooMany); + wrapper.appendChild(tooMany2); + return wrapper; + } + + const list = document.createElement("ul"); + list.classList.add("app-popup-list"); + features.forEach((feature) => { + const featureType = capitalizeFirstLetter( + feature.sourceLayer || feature.source + ).replaceAll("-", " "); + const fillColour = this.getFillColour(feature); + + const featureName = + feature.properties.name || feature.properties.reference || "Not Named"; + const item = document.createElement("li"); + item.classList.add("app-popup-item"); + item.style.borderLeft = `5px solid ${fillColour}`; + + const secondaryText = document.createElement("p"); + secondaryText.classList.add( + "app-u-secondary-text", + "govuk-!-margin-bottom-0", + "govuk-!-margin-top-0" + ); + secondaryText.textContent = featureType; + item.appendChild(secondaryText); + + const link = document.createElement("a"); + link.classList.add("govuk-link"); + link.href = `/entity/${feature.properties.entity}`; + link.textContent = featureName; + const smallText = document.createElement("p"); + smallText.classList.add( + "dl-small-text", + "govuk-!-margin-top-0", + "govuk-!-margin-bottom-0" + ); + smallText.appendChild(link); + item.appendChild(smallText); + + list.appendChild(item); + }); + + wrapper.appendChild(list); + return wrapper; + } + + getFillColour(feature) { + if (feature.layer.type === "symbol") + return this.map.getLayer(feature.layer.id).getPaintProperty("icon-color"); + else if (feature.layer.type === "fill") + return this.map.getLayer(feature.layer.id).getPaintProperty("fill-color"); + else if (feature.layer.type === "fill-extrusion") + return this.map + .getLayer(feature.layer.id) + .getPaintProperty("fill-extrusion-color"); + else if (feature.layer.type === "circle") + return this.map + .getLayer(feature.layer.id) + .getPaintProperty("circle-color"); + else + throw new Error( + "could not get fill colour for feature of type " + feature.layer.type + ); + } + + setLayerVisibility(layerName, visibility) { + this.map.setLayoutProperty(layerName, "visibility", visibility); + } +} diff --git a/assets/javascripts/utils.js b/assets/javascripts/utils.js index 8c1cad77..5f034e0f 100644 --- a/assets/javascripts/utils.js +++ b/assets/javascripts/utils.js @@ -1,57 +1,57 @@ -import MapController from "./MapController.js"; - -export const newMapController = (params = { layers: [] }) => { - const datasetUrl = params.DATASETTE_TILES_URL || ""; - - let mapParams = { - ...params, - vectorSource: `${datasetUrl}/dataset_tiles/{z}/{x}/{y}.vector.pbf`, - datasetVectorUrl: `${datasetUrl}/`, - datasets: params.layers.map((d) => d.dataset), - sources: params.layers.map((d) => { - return { - name: d.dataset + "-source", - vectorSource: `${datasetUrl}/${d.dataset}/{z}/{x}/{y}.vector.pbf`, - }; - }), - mapId: params.mapId || "map", - }; - return new MapController(mapParams); -}; - -export const capitalizeFirstLetter = (string) => { - return string.charAt(0).toUpperCase() + string.slice(1); -}; - -export const convertNodeListToArray = (nl) => { - return Array.prototype.slice.call(nl); -}; - -// Prevents scrolling of the page when the user triggers the wheel event on a div -// while still allowing scrolling of any specified scrollable child elements. -// Params: -// scrollableChildElements: an array of class names of potential scrollable elements -export const preventScroll = (scrollableChildElements = []) => { - return (e) => { - const closestClassName = scrollableChildElements.find((c) => { - return e.target.closest(c) != null; - }); - - if (!closestClassName) { - e.preventDefault(); - return false; - } - - const list = e.target.closest(closestClassName); - - if (!list) { - e.preventDefault(); - return false; - } - - var verticalScroll = list.scrollHeight > list.clientHeight; - if (!verticalScroll) e.preventDefault(); - - return false; - }; -}; +import MapController from "./MapController.js"; + +export const newMapController = (params = { layers: [] }) => { + const datasetUrl = params.DATASETTE_TILES_URL || ""; + + let mapParams = { + ...params, + vectorSource: `${datasetUrl}/tiles/{z}/{x}/{y}.vector.pbf`, + datasetVectorUrl: `${datasetUrl}/`, + datasets: params.layers.map((d) => d.dataset), + sources: params.layers.map((d) => { + return { + name: d.dataset + "-source", + vectorSource: `${datasetUrl}/tiles/${d.dataset}/{z}/{x}/{y}.vector.pbf`, + }; + }), + mapId: params.mapId || "map", + }; + return new MapController(mapParams); +}; + +export const capitalizeFirstLetter = (string) => { + return string.charAt(0).toUpperCase() + string.slice(1); +}; + +export const convertNodeListToArray = (nl) => { + return Array.prototype.slice.call(nl); +}; + +// Prevents scrolling of the page when the user triggers the wheel event on a div +// while still allowing scrolling of any specified scrollable child elements. +// Params: +// scrollableChildElements: an array of class names of potential scrollable elements +export const preventScroll = (scrollableChildElements = []) => { + return (e) => { + const closestClassName = scrollableChildElements.find((c) => { + return e.target.closest(c) != null; + }); + + if (!closestClassName) { + e.preventDefault(); + return false; + } + + const list = e.target.closest(closestClassName); + + if (!list) { + e.preventDefault(); + return false; + } + + var verticalScroll = list.scrollHeight > list.clientHeight; + if (!verticalScroll) e.preventDefault(); + + return false; + }; +}; From d18576b0752515984a881338b08e8e1a37a7c2f2 Mon Sep 17 00:00:00 2001 From: James Bannister Date: Thu, 9 May 2024 10:57:53 +0100 Subject: [PATCH 10/16] Geojson returns --- application/routers/tiles.py | 94 ++++++++---------------------------- 1 file changed, 20 insertions(+), 74 deletions(-) diff --git a/application/routers/tiles.py b/application/routers/tiles.py index 0bc0b369..5c4371e9 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -1,76 +1,25 @@ -from fastapi import APIRouter, HTTPException, Depends, Request -from fastapi.responses import StreamingResponse +from fastapi import APIRouter, HTTPException, Depends from sqlalchemy.orm import Session -from sqlalchemy import func, case, and_ -from io import BytesIO - from application.db.models import EntityOrm from application.db.session import get_session router = APIRouter() -# ============================================================ -# Helper Funcs -# ============================================================ - -# Validate tile x/y coordinates at the given zoom level -def tile_is_valid(tile): - size = 2 ** tile["zoom"] - return ( - 0 <= tile["x"] < size - and 0 <= tile["y"] < size - and tile["format"] in ["pbf", "mvt"] - ) +def tile_is_valid(z, x, y, fmt): + max_tile = 2**z - 1 + return 0 <= x <= max_tile and 0 <= y <= max_tile and fmt in ["pbf", "mvt"] -# Build the database query using SQLAlchemy ORM to match the direct SQL logic def build_db_query(tile, session: Session): - srid = 4326 # WGS 84 - - # Define the envelope, webmercator, and WGS84 transformations - envelope = func.ST_TileEnvelope(tile["zoom"], tile["x"], tile["y"]) - webmercator = envelope - wgs84 = func.ST_Transform(webmercator, srid) - bounds = func.ST_MakeEnvelope(-180, -85.0511287798066, 180, 85.0511287798066, srid) - - # Define the CASE expression for geometry transformation - geometry_case = case( - [ - ( - func.ST_Covers(bounds, EntityOrm.geometry), - func.ST_Transform(EntityOrm.geometry, srid), - ) - ], - else_=func.ST_Transform(func.ST_Intersection(bounds, EntityOrm.geometry), srid), - ) - - # Geometry processing for MVT - query = ( - session.query(func.ST_AsMVTGeom(geometry_case, wgs84).label("geom")) - .filter( - and_( - func.ST_Intersects(EntityOrm.geometry, wgs84), - EntityOrm.dataset == tile["dataset"], - ) - ) - .subquery() - ) - - # Generate MVT from a single-column subquery - tile_data = session.query(func.ST_AsMVT(query.c.geom, tile["dataset"])).scalar() - - return tile_data - - -# ============================================================ -# API Endpoints -# ============================================================ + # z, x, y, dataset = tile["zoom"], tile["x"], tile["y"], tile["dataset"] + dataset = tile["dataset"] + geometries = session.query(EntityOrm).filter(EntityOrm.dataset == dataset).all() + return geometries @router.get("/{dataset}/{z}/{x}/{y}.vector.{fmt}") -async def read_tiles_from_postgres( - request: Request, +async def read_tiles( dataset: str, z: int, x: int, @@ -78,21 +27,18 @@ async def read_tiles_from_postgres( fmt: str, session: Session = Depends(get_session), ): - print("Hello", {dataset}) - tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} - if not tile_is_valid(tile): - raise HTTPException(status_code=400, detail=f"Invalid tile path: {tile}") + if not tile_is_valid(z, x, y, fmt): + raise HTTPException(status_code=400, detail="Invalid tile path") - tile_data = build_db_query(tile, session) - if not tile_data: + tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} + geometries = build_db_query(tile, session) + if not geometries: raise HTTPException(status_code=404, detail="Tile data not found") - pbf_buffer = BytesIO(tile_data) - resp_headers = { - "Access-Control-Allow-Origin": "*", - "Content-Type": "application/vnd.mapbox-vector-tile", - } + geojson_features = [ + {"type": "Feature", "geometry": geom.geojson} for geom in geometries + ] + + geojson_data = {"type": "FeatureCollection", "features": geojson_features} - return StreamingResponse( - pbf_buffer, media_type="vnd.mapbox-vector-tile", headers=resp_headers - ) + return geojson_data From 0bf5c05671f06dbd7730f10c57e147a0f3273c80 Mon Sep 17 00:00:00 2001 From: James Bannister Date: Thu, 9 May 2024 11:12:16 +0100 Subject: [PATCH 11/16] Integrate Zoom --- application/routers/tiles.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/application/routers/tiles.py b/application/routers/tiles.py index 5c4371e9..2b90fad8 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -1,5 +1,7 @@ from fastapi import APIRouter, HTTPException, Depends from sqlalchemy.orm import Session +from sqlalchemy import func +import math from application.db.models import EntityOrm from application.db.session import get_session @@ -11,10 +13,31 @@ def tile_is_valid(z, x, y, fmt): return 0 <= x <= max_tile and 0 <= y <= max_tile and fmt in ["pbf", "mvt"] +def tile_bounds(z, x, y): + n = 2.0**z + lon_min = x / n * 360.0 - 180.0 + lat_min = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * y / n)))) + lon_max = (x + 1) / n * 360.0 - 180.0 + lat_max = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * (y + 1) / n)))) + return lon_min, lat_min, lon_max, lat_max + + def build_db_query(tile, session: Session): - # z, x, y, dataset = tile["zoom"], tile["x"], tile["y"], tile["dataset"] - dataset = tile["dataset"] - geometries = session.query(EntityOrm).filter(EntityOrm.dataset == dataset).all() + z, x, y, dataset = tile["zoom"], tile["x"], tile["y"], tile["dataset"] + lon_min, lat_min, lon_max, lat_max = tile_bounds(z, x, y) + + # Using ST_MakeEnvelope to create a bounding box and filtering points within it + bounding_box = func.ST_MakeEnvelope(lon_min, lat_min, lon_max, lat_max, 4326) + + # Query geometries that intersect with the bounding box + geometries = ( + session.query(EntityOrm) + .filter( + EntityOrm.dataset == dataset, + func.ST_Intersects(EntityOrm.point, bounding_box), + ) + .all() + ) return geometries From dcfee9ce14b4b9448fb2a37c097523bd26a27d94 Mon Sep 17 00:00:00 2001 From: James Bannister Date: Thu, 9 May 2024 12:06:48 +0100 Subject: [PATCH 12/16] Move back to MVT --- application/routers/tiles.py | 60 ++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/application/routers/tiles.py b/application/routers/tiles.py index 2b90fad8..b7e98a14 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -1,8 +1,7 @@ -from fastapi import APIRouter, HTTPException, Depends +from fastapi import APIRouter, HTTPException, Depends, Response from sqlalchemy.orm import Session -from sqlalchemy import func import math -from application.db.models import EntityOrm +from sqlalchemy import text from application.db.session import get_session router = APIRouter() @@ -26,19 +25,38 @@ def build_db_query(tile, session: Session): z, x, y, dataset = tile["zoom"], tile["x"], tile["y"], tile["dataset"] lon_min, lat_min, lon_max, lat_max = tile_bounds(z, x, y) - # Using ST_MakeEnvelope to create a bounding box and filtering points within it - bounding_box = func.ST_MakeEnvelope(lon_min, lat_min, lon_max, lat_max, 4326) + geometry_column = "geometry" + if dataset == "conservation-area": + geometry_column = "point" - # Query geometries that intersect with the bounding box - geometries = ( - session.query(EntityOrm) - .filter( - EntityOrm.dataset == dataset, - func.ST_Intersects(EntityOrm.point, bounding_box), - ) - .all() + tile_width = 256 + + mvt_geom_query = text( + f""" + SELECT ST_AsMVT(q, 'entities_layer', {tile_width}, 'geom') AS mvt + FROM ( + SELECT ST_AsMVTGeom( + {geometry_column}, + ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326), + {tile_width}, 4096, true) AS geom + FROM entity + WHERE dataset = :dataset + AND ST_Intersects({geometry_column}, ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326)) + ) AS q + """ ) - return geometries + + result = session.execute( + mvt_geom_query, + { + "lon_min": lon_min, + "lat_min": lat_min, + "lon_max": lon_max, + "lat_max": lat_max, + "dataset": dataset, + }, + ).scalar() + return result @router.get("/{dataset}/{z}/{x}/{y}.vector.{fmt}") @@ -54,14 +72,10 @@ async def read_tiles( raise HTTPException(status_code=400, detail="Invalid tile path") tile = {"dataset": dataset, "zoom": z, "x": x, "y": y, "format": fmt} - geometries = build_db_query(tile, session) - if not geometries: + mvt_data = build_db_query(tile, session) + if not mvt_data: raise HTTPException(status_code=404, detail="Tile data not found") - geojson_features = [ - {"type": "Feature", "geometry": geom.geojson} for geom in geometries - ] - - geojson_data = {"type": "FeatureCollection", "features": geojson_features} - - return geojson_data + return Response( + content=mvt_data.tobytes(), media_type="application/vnd.mapbox-vector-tile" + ) From bbdc5993a5673cf44a0414e71573ffb4f041db3b Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Thu, 16 May 2024 15:37:56 +0100 Subject: [PATCH 13/16] building tile vector using ST_ASMVT --- application/routers/tiles.py | 51 +++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/application/routers/tiles.py b/application/routers/tiles.py index b7e98a14..949b9d78 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -26,25 +26,49 @@ def build_db_query(tile, session: Session): lon_min, lat_min, lon_max, lat_max = tile_bounds(z, x, y) geometry_column = "geometry" - if dataset == "conservation-area": - geometry_column = "point" tile_width = 256 mvt_geom_query = text( - f""" - SELECT ST_AsMVT(q, 'entities_layer', {tile_width}, 'geom') AS mvt - FROM ( - SELECT ST_AsMVTGeom( - {geometry_column}, - ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326), - {tile_width}, 4096, true) AS geom - FROM entity - WHERE dataset = :dataset - AND ST_Intersects({geometry_column}, ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326)) + f"""SELECT ST_AsMVT(q, :dataset, :tile_width, 'geom') FROM + (SELECT + ST_AsMVTGeom( + {geometry_column}, + ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326), + :tile_width, + 4096, + true + ) as geom, + jsonb_build_object( + 'name', entity.name, + 'dataset', entity.dataset, + 'organisation-entity', entity.organisation_entity, + 'entity', entity.entity, + 'entry-date', entity.entry_date, + 'start-date', entity.start_date, + 'end-date', entity.end_date, + 'prefix', entity.prefix, + 'reference', entity.reference + ) AS properties + FROM entity + WHERE dataset = :dataset + AND ST_Intersects({geometry_column}, ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326)) ) AS q - """ + """ ) + # f""" + # SELECT ST_AsMVT(q, 'entities_layer', {tile_width}, 'geom') AS mvt + # FROM ( + # SELECT ST_AsMVTGeom( + # {geometry_column}, + # ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326), + # {tile_width}, 4096, true) AS geom + # FROM entity + # WHERE dataset = :dataset + # AND ST_Intersects({geometry_column}, ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326)) + # ) AS q + # """ + # ) result = session.execute( mvt_geom_query, @@ -54,6 +78,7 @@ def build_db_query(tile, session: Session): "lon_max": lon_max, "lat_max": lat_max, "dataset": dataset, + "tile_width": tile_width, }, ).scalar() return result From fc85089ef4c084d2195d5f4ca0b765039db8f15d Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Thu, 16 May 2024 15:50:12 +0100 Subject: [PATCH 14/16] Adding point geometry for tree --- application/routers/tiles.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/application/routers/tiles.py b/application/routers/tiles.py index 949b9d78..abe0d7b8 100644 --- a/application/routers/tiles.py +++ b/application/routers/tiles.py @@ -26,6 +26,8 @@ def build_db_query(tile, session: Session): lon_min, lat_min, lon_max, lat_max = tile_bounds(z, x, y) geometry_column = "geometry" + if dataset == "tree": + geometry_column = "point" tile_width = 256 @@ -51,24 +53,15 @@ def build_db_query(tile, session: Session): 'reference', entity.reference ) AS properties FROM entity - WHERE dataset = :dataset + WHERE NOT EXISTS ( + SELECT 1 FROM old_entity + WHERE entity.entity = old_entity.old_entity + ) + AND dataset = :dataset AND ST_Intersects({geometry_column}, ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326)) ) AS q """ ) - # f""" - # SELECT ST_AsMVT(q, 'entities_layer', {tile_width}, 'geom') AS mvt - # FROM ( - # SELECT ST_AsMVTGeom( - # {geometry_column}, - # ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326), - # {tile_width}, 4096, true) AS geom - # FROM entity - # WHERE dataset = :dataset - # AND ST_Intersects({geometry_column}, ST_MakeEnvelope(:lon_min, :lat_min, :lon_max, :lat_max, 4326)) - # ) AS q - # """ - # ) result = session.execute( mvt_geom_query, From aa6c8b6059d94f2789da353de1769ea68ddf5b72 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Fri, 17 May 2024 14:06:14 +0100 Subject: [PATCH 15/16] Overwriting DATASETTE_TILES_URL for testing --- application/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/settings.py b/application/settings.py index aa407dc1..04000999 100644 --- a/application/settings.py +++ b/application/settings.py @@ -16,7 +16,7 @@ class Settings(BaseSettings): RELEASE_TAG: Optional[str] = None ENVIRONMENT: str DATASETTE_URL: HttpUrl - DATASETTE_TILES_URL: Optional[HttpUrl] + DATASETTE_TILES_URL: Optional[HttpUrl] = "https://www.development.digital-land.info" DATA_FILE_URL: HttpUrl GA_MEASUREMENT_ID: Optional[str] = None OS_CLIENT_KEY: Optional[str] = None @@ -28,6 +28,7 @@ def get_settings() -> Settings: # TODO remove as Gov PaaS is no longer needed # Gov.uk PaaS provides a URL to the postgres instance it provisions via DATABASE_URL # See https://docs.cloud.service.gov.uk/deploying_services/postgresql/#connect-to-a-postgresql-service-from-your-app + if "DATABASE_URL" in os.environ: database_url = os.environ["DATABASE_URL"].replace( "postgres://", "postgresql://", 1 From fb841dff2d2d72055962421e1f87e20a11bf3eae Mon Sep 17 00:00:00 2001 From: James Bannister Date: Fri, 17 May 2024 14:30:17 +0100 Subject: [PATCH 16/16] Hardcode URL test --- application/settings.py | 1 - .../templates/components/map/macro.jinja | 6 +- application/templates/national-map.html | 87 ++++++++----------- assets/javascripts/utils.js | 2 +- 4 files changed, 38 insertions(+), 58 deletions(-) diff --git a/application/settings.py b/application/settings.py index 04000999..ea6bc3b0 100644 --- a/application/settings.py +++ b/application/settings.py @@ -16,7 +16,6 @@ class Settings(BaseSettings): RELEASE_TAG: Optional[str] = None ENVIRONMENT: str DATASETTE_URL: HttpUrl - DATASETTE_TILES_URL: Optional[HttpUrl] = "https://www.development.digital-land.info" DATA_FILE_URL: HttpUrl GA_MEASUREMENT_ID: Optional[str] = None OS_CLIENT_KEY: Optional[str] = None diff --git a/application/templates/components/map/macro.jinja b/application/templates/components/map/macro.jinja index 4521f82a..dac4ce8e 100644 --- a/application/templates/components/map/macro.jinja +++ b/application/templates/components/map/macro.jinja @@ -53,14 +53,14 @@ params = { ...params, baseTileStyleFilePath: "/static/javascripts/base-tile.json", - vectorSource: "{{ params.DATASETTE_TILES_URL }}/tiles/{z}/{x}/{y}.vector.pbf", - datasetVectorUrl: "{{ params.DATASETTE_TILES_URL }}/tiles/", + vectorSource: "https://www.development.digital-land.info/tiles/{z}/{x}/{y}.vector.pbf", + datasetVectorUrl: "https://www.development.digital-land.info/tiles/", datasets: {{layers|tojson}}.map(d => d.dataset), vectorTileSources: {{layers|tojson}}.map(d => { d.paint_options = d.paint_options || {}; return { name: d.dataset, - vectorSource: "{{ params.DATASETTE_TILES_URL }}/tiles/" + d.dataset + "/{z}/{x}/{y}.vector.pbf", + vectorSource: "https://www.development.digital-land.info/tiles/" + d.dataset + "/{z}/{x}/{y}.vector.pbf", dataType: d.paint_options.type, styleProps: { colour: d.paint_options.colour, diff --git a/application/templates/national-map.html b/application/templates/national-map.html index 24aee274..3857b2c7 100644 --- a/application/templates/national-map.html +++ b/application/templates/national-map.html @@ -1,58 +1,39 @@ -{% extends "layouts/layout--full-width.html" %} -{% set templateName = "dl-info/national-map.html" %} - -{%- from "components/map/macro.jinja" import map %} - -{% set containerClasses = 'dl-container--full-width' %} -{% set fullWidthHeader = true %} - -{% set includesMap = true %} -{% block pageTitle %}Map of planning data for England | Planning Data{% endblock %} - -{% - set notePanel = '

Find, understand and download the datasets used to create this map.

' -%} - -{%- block mapAssets %} - - - {{ super() }} -{% endblock -%} - -{%- from "components/back-button/macro.jinja" import dlBackButton %} -{% block breadcrumbs%} - {{ dlBackButton({ - "parentHref": '/' - })}} -{% endblock %} - -{% block content %} -
-
-

Map of planning data for England

-
+{% extends "layouts/layout--full-width.html" %} {% set templateName = +"dl-info/national-map.html" %} {%- from "components/map/macro.jinja" import map +%} {% set containerClasses = 'dl-container--full-width' %} {% set +fullWidthHeader = true %} {% set includesMap = true %} {% block pageTitle %}Map +of planning data for England | Planning Data{% endblock %} {% set notePanel = ' +

+ Find, understand and download the + datasets used to create this map. +

+' %} {%- block mapAssets %} + + +{{ super() }} {% endblock -%} {%- from "components/back-button/macro.jinja" +import dlBackButton %} {% block breadcrumbs%} {{ dlBackButton({ "parentHref": +'/' })}} {% endblock %} {% block content %} +
+
+

+ Map of planning data for England +

+
-

See the data we've collected and collated on a map.

- - {{ - map({ - 'height': 700, - 'layers': layers, - 'DATASETTE_TILES_URL': settings.DATASETTE_TILES_URL, - 'notePanel': notePanel, - 'enableZoomControls': true, - 'enableLayerControls': true, - 'enableZoomCounter': true, - }) - }} - -

This prototype map is automatically created using data from planning.data.gov.uk. Find out more about the Planning Data Platform

- -{% endblock %} +

+ See the data we've collected and collated on a map. +

-{% block bodyEnd %} -{{ super() }} +{{ map({ 'height': 700, 'layers': layers, 'DATASETTE_TILES_URL': +"https://www.development.digital-land.info", 'notePanel': notePanel, +'enableZoomControls': true, 'enableLayerControls': true, 'enableZoomCounter': +true, }) }} +

+ This prototype map is automatically created using data from + planning.data.gov.uk. Find out more + about the Planning Data Platform +

-{% endblock %} +{% endblock %} {% block bodyEnd %} {{ super() }} {% endblock %} diff --git a/assets/javascripts/utils.js b/assets/javascripts/utils.js index 5f034e0f..3a21b1eb 100644 --- a/assets/javascripts/utils.js +++ b/assets/javascripts/utils.js @@ -1,7 +1,7 @@ import MapController from "./MapController.js"; export const newMapController = (params = { layers: [] }) => { - const datasetUrl = params.DATASETTE_TILES_URL || ""; + const datasetUrl = "https://www.development.digital-land.info"; let mapParams = { ...params,