From 024a279061a614301e0890dc2305e0c0c523e660 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 29 Mar 2018 01:26:43 +0200 Subject: [PATCH 01/28] Generating a simple taginfo project file from "openstreetmap-carto.style"; "project.mml" --- taginfo-project/README.md | 37 + .../generate-taginfo-project-file.py | 130 ++++ .../taginfo-openstreetmap-carto.json | 699 ++++++++++++++++++ 3 files changed, 866 insertions(+) create mode 100644 taginfo-project/README.md create mode 100755 taginfo-project/generate-taginfo-project-file.py create mode 100644 taginfo-project/taginfo-openstreetmap-carto.json diff --git a/taginfo-project/README.md b/taginfo-project/README.md new file mode 100644 index 0000000000..fd5dfe03e7 --- /dev/null +++ b/taginfo-project/README.md @@ -0,0 +1,37 @@ + + +The taginfo database keeps the information which projects use which OSM keys and tags +* site: https://taginfo.openstreetmap.org +* wiki: https://wiki.openstreetmap.org/wiki/Taginfo + +Now We can generate only a minimal info - about the used 'keys' +see more https://github.com/gravitystorm/openstreetmap-carto/issues/961 + +### WHEN to run? +* when the `../openstreetmap-carto.style` or `../project.mml` change , this is the 2 input files for detecting osm keys + +### HOW to run: +* from this directory: `python3 ./generate-taginfo-project-file.py` + +### RESULT: +* the new: `taginfo-openstreetmap-carto.json` ( and some debug info in the screen! ) + +### Known limitations +* Only a subset of hstore `tags->` is parsed from the `../project.mml` +* Don't work the hstore ARRAY parsing: `tags @> ARRAY['building:level','building:color']` TODO +* This code tested only on Ubuntu Linux +* Check the result! + +### How to debug: +The taginfo project_list file should contain a link to this repo ( /taginfo-project/taginfo-openstreetmap-carto.json ) +* https://github.com/taginfo/taginfo-projects/blob/master/project_list.txt + +the expected line: +* openstreetmap_carto https://raw.githubusercontent.com/gravitystorm/openstreetmap-carto/master/taginfo-project/taginfo-openstreetmap-carto.json + +After the daily refresh the project info should be find here: +* https://taginfo.openstreetmap.org/projects/openstreetmap_carto + +### Disclaimer and Attribution : This code is based on +* Paul Norman code : https://github.com/osmlab/osm2pgsql_taginfo +* Sven Geggus code : https://github.com/giggls/openstreetmap-carto-de/blob/master/views_osmde/generate_taginfo.py diff --git a/taginfo-project/generate-taginfo-project-file.py b/taginfo-project/generate-taginfo-project-file.py new file mode 100755 index 0000000000..cb004af383 --- /dev/null +++ b/taginfo-project/generate-taginfo-project-file.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 + +# ------------------------------------------------------------------------------- +# +# This code generate a taginfo project list file (see more https://wiki.openstreetmap.org/wiki/Taginfo/Projects ) +# +### Attribution & Disclaimer: +# This code is based on : +# Paul Norman code : https://github.com/osmlab/osm2pgsql_taginfo +# Sven Geggus code : https://github.com/giggls/openstreetmap-carto-de/blob/master/views_osmde/generate_taginfo.py +# ------------------------------------------------------- + +import re +import json +import yaml + +taginfo = { + "data_format": 1, + "project": { + "name": "openstreetmap.org carto keys", + "description": "OpenStreetMap.org mapnik style, in CartoCSS", + "project_url": "https://github.com/gravitystorm/openstreetmap-carto", + "contact_name": "openstreetmap-carto maintainers", + "contact_email": "openstreetmap-carto (at) gravitystorm (dot) co (dot) uk" + }, + "tags": [] +} + + +# +# Parsing openstreetmap-carto.style file +# +with open('../openstreetmap-carto.style', 'r') as style: + for line in style: + if line[0] == '#': + continue + keyline = line.split() + if len(keyline) != 4: + continue + if keyline[3] == 'delete' or 'nocolumn' in keyline[3]: + continue + key = keyline[1] + object_types = [] + if 'node' in keyline[0]: + object_types.append('node') + if 'way' in keyline[0]: + object_types.append('way') + if 'polygon' in keyline[3]: + object_types.append('area') + + if ('area' in object_types) or ('way' in object_types ): + object_types.append('relation') + + if key not in ('z_order','way_area'): + taginfo["tags"].append( + { + "key": key, + "object_types": object_types, + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q="+key + }) + + + +# +# Parsing "project.mml" file for the HSTORE keys ( tags-> ) +# + +with open('../project.mml', 'r') as f: + newf = yaml.load(f.read()) +f.closed + + +tags_b = re.compile(r"tags[^'^)]*'.+?'") + +# ["tags @> 'capital=>yes'"] +# ["tags ? 'wetland'", "tags->'wetland'", "tags->'leaf_type'"] +# (tags @> '"generator:source"=>wind' + +allhstoretags={} + +for layer in newf["Layer"]: + print( "########### processing Layer: ", layer["id"]," ###########" ) + ds_geometry = layer.get("geometry") + + osmtype = '' + if ds_geometry: + if ds_geometry=='point': + osmtype='node' + elif ds_geometry=='linestring': + osmtype='way' + elif ds_geometry=='polygon': + osmtype='area' + + ds_type = layer["Datasource"].get("type") + if ds_type and ds_type == "postgis": + ds_table = layer["Datasource"].get("table") + if ds_table: + tags01 = tags_b.findall(ds_table) + if tags01: + print(tags01) + for tag in tags01: + key=tag.split("'")[1].split("=")[0].replace('"','') + if key: + print("--:", ds_geometry,"->", osmtype, " key:", key) + if key not in allhstoretags: + k = [ osmtype ] + allhstoretags[key]=k + elif osmtype not in allhstoretags[key]: + allhstoretags[key].append(osmtype) + +for k in allhstoretags: + # add "relation" if "area" or "way" + if ('area' in allhstoretags[k]) or ('way' in allhstoretags[k]): + allhstoretags[k].append("relation") + + taginfo["tags"].append( + { + "key": k, + "object_types": allhstoretags[k], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q="+k + }) + + + +with open('taginfo-openstreetmap-carto.json', 'w') as outfile: + json.dump(taginfo, outfile, indent=4) + + \ No newline at end of file diff --git a/taginfo-project/taginfo-openstreetmap-carto.json b/taginfo-project/taginfo-openstreetmap-carto.json new file mode 100644 index 0000000000..7f67c881e7 --- /dev/null +++ b/taginfo-project/taginfo-openstreetmap-carto.json @@ -0,0 +1,699 @@ +{ + "data_format": 1, + "project": { + "name": "openstreetmap.org carto keys", + "description": "OpenStreetMap.org mapnik style, in CartoCSS", + "project_url": "https://github.com/gravitystorm/openstreetmap-carto", + "contact_name": "openstreetmap-carto maintainers", + "contact_email": "openstreetmap-carto (at) gravitystorm (dot) co (dot) uk" + }, + "tags": [ + { + "key": "access", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=access" + }, + { + "key": "addr:housename", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=addr:housename" + }, + { + "key": "addr:housenumber", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=addr:housenumber" + }, + { + "key": "addr:interpolation", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=addr:interpolation" + }, + { + "key": "admin_level", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=admin_level" + }, + { + "key": "aerialway", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=aerialway" + }, + { + "key": "aeroway", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=aeroway" + }, + { + "key": "amenity", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=amenity" + }, + { + "key": "barrier", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=barrier" + }, + { + "key": "bicycle", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=bicycle" + }, + { + "key": "bridge", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=bridge" + }, + { + "key": "boundary", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=boundary" + }, + { + "key": "building", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=building" + }, + { + "key": "construction", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=construction" + }, + { + "key": "covered", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=covered" + }, + { + "key": "foot", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=foot" + }, + { + "key": "highway", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=highway" + }, + { + "key": "historic", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=historic" + }, + { + "key": "horse", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=horse" + }, + { + "key": "junction", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=junction" + }, + { + "key": "landuse", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=landuse" + }, + { + "key": "layer", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=layer" + }, + { + "key": "leisure", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=leisure" + }, + { + "key": "lock", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=lock" + }, + { + "key": "man_made", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=man_made" + }, + { + "key": "military", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=military" + }, + { + "key": "name", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=name" + }, + { + "key": "natural", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=natural" + }, + { + "key": "oneway", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=oneway" + }, + { + "key": "place", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=place" + }, + { + "key": "power", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=power" + }, + { + "key": "railway", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=railway" + }, + { + "key": "ref", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ref" + }, + { + "key": "religion", + "object_types": [ + "node", + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=religion" + }, + { + "key": "route", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=route" + }, + { + "key": "service", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=service" + }, + { + "key": "shop", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=shop" + }, + { + "key": "surface", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=surface" + }, + { + "key": "tourism", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tourism" + }, + { + "key": "tracktype", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tracktype" + }, + { + "key": "tunnel", + "object_types": [ + "way", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tunnel" + }, + { + "key": "water", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=water" + }, + { + "key": "waterway", + "object_types": [ + "node", + "way", + "area", + "relation" + ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=waterway" + }, + { + "key": "wetland", + "object_types": [ + "area", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=wetland" + }, + { + "key": "intermittent", + "object_types": [ + "way", + "area", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=intermittent" + }, + { + "key": "seasonal", + "object_types": [ + "way", + "area", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=seasonal" + }, + { + "key": "leaf_type", + "object_types": [ + "area", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=leaf_type" + }, + { + "key": "public_transport", + "object_types": [ + "area", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=public_transport" + }, + { + "key": "entrance", + "object_types": [ + "node" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=entrance" + }, + { + "key": "indoor", + "object_types": [ + "node" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=indoor" + }, + { + "key": "population", + "object_types": [ + "node" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=population" + }, + { + "key": "capital", + "object_types": [ + "node" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=capital" + }, + { + "key": "advertising", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=advertising" + }, + { + "key": "memorial", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=memorial" + }, + { + "key": "denomination", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=denomination" + }, + { + "key": "generator:source", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=generator:source" + }, + { + "key": "height", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=height" + }, + { + "key": "power_source", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=power_source" + }, + { + "key": "icao", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=icao" + }, + { + "key": "iata", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=iata" + }, + { + "key": "recycling_type", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=recycling_type" + }, + { + "key": "tower:construction", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tower:construction" + }, + { + "key": "tower:type", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tower:type" + }, + { + "key": "ford", + "object_types": [ + "way", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ford" + }, + { + "key": "emergency", + "object_types": [ + "node" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=emergency" + }, + { + "key": "ele", + "object_types": [ + "node", + "area", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ele" + }, + { + "key": "highspeed", + "object_types": [ + "way", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=highspeed" + }, + { + "key": "usage", + "object_types": [ + "way", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=usage" + }, + { + "key": "operator", + "object_types": [ + "area", + "way", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=operator" + }, + { + "key": "addr:unit", + "object_types": [ + "node" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=addr:unit" + } + ] +} \ No newline at end of file From 54b37391b4e473ec3fa00be7fe02de590197a0e3 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 29 Mar 2018 20:44:25 +0200 Subject: [PATCH 02/28] replace project name to "OpenStreetMap Carto keys" --- taginfo-project/generate-taginfo-project-file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taginfo-project/generate-taginfo-project-file.py b/taginfo-project/generate-taginfo-project-file.py index cb004af383..f2932db02b 100755 --- a/taginfo-project/generate-taginfo-project-file.py +++ b/taginfo-project/generate-taginfo-project-file.py @@ -17,7 +17,7 @@ taginfo = { "data_format": 1, "project": { - "name": "openstreetmap.org carto keys", + "name": "OpenStreetMap Carto keys", "description": "OpenStreetMap.org mapnik style, in CartoCSS", "project_url": "https://github.com/gravitystorm/openstreetmap-carto", "contact_name": "openstreetmap-carto maintainers", From 0b32ba4faf2cc01e854ccbf321ccfe678b7be2d0 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 29 Mar 2018 20:45:54 +0200 Subject: [PATCH 03/28] regenerated - taginfo project file --- taginfo-project/taginfo-openstreetmap-carto.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taginfo-project/taginfo-openstreetmap-carto.json b/taginfo-project/taginfo-openstreetmap-carto.json index 7f67c881e7..f1acd4774f 100644 --- a/taginfo-project/taginfo-openstreetmap-carto.json +++ b/taginfo-project/taginfo-openstreetmap-carto.json @@ -1,7 +1,7 @@ { "data_format": 1, "project": { - "name": "openstreetmap.org carto keys", + "name": "OpenStreetMap Carto keys", "description": "OpenStreetMap.org mapnik style, in CartoCSS", "project_url": "https://github.com/gravitystorm/openstreetmap-carto", "contact_name": "openstreetmap-carto maintainers", From 4e87eb952760b4d0ee29d36478433e2343692155 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 29 Mar 2018 21:33:59 +0200 Subject: [PATCH 04/28] change description to "Default OpenStreetMap.org style using CartoCSS" --- taginfo-project/generate-taginfo-project-file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taginfo-project/generate-taginfo-project-file.py b/taginfo-project/generate-taginfo-project-file.py index f2932db02b..dab0711855 100755 --- a/taginfo-project/generate-taginfo-project-file.py +++ b/taginfo-project/generate-taginfo-project-file.py @@ -18,7 +18,7 @@ "data_format": 1, "project": { "name": "OpenStreetMap Carto keys", - "description": "OpenStreetMap.org mapnik style, in CartoCSS", + "description": "Default OpenStreetMap.org style using CartoCSS", "project_url": "https://github.com/gravitystorm/openstreetmap-carto", "contact_name": "openstreetmap-carto maintainers", "contact_email": "openstreetmap-carto (at) gravitystorm (dot) co (dot) uk" From 4deb542aadbb5951d1eebf7b19174d93f658ab79 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Fri, 30 Mar 2018 23:15:51 +0200 Subject: [PATCH 05/28] some refactoring in parsing --- taginfo-project/README.md | 17 +++- .../generate-taginfo-project-file.py | 96 ++++++++++++++----- .../taginfo-openstreetmap-carto.json | 2 +- 3 files changed, 84 insertions(+), 31 deletions(-) diff --git a/taginfo-project/README.md b/taginfo-project/README.md index fd5dfe03e7..097b407032 100644 --- a/taginfo-project/README.md +++ b/taginfo-project/README.md @@ -1,13 +1,13 @@ -The taginfo database keeps the information which projects use which OSM keys and tags +The taginfo database keeps the information which projects use which OSM keys and tags * site: https://taginfo.openstreetmap.org * wiki: https://wiki.openstreetmap.org/wiki/Taginfo -Now We can generate only a minimal info - about the used 'keys' +Now We can generate only a minimal info - about the used 'keys' see more https://github.com/gravitystorm/openstreetmap-carto/issues/961 -### WHEN to run? +### WHEN to run? * when the `../openstreetmap-carto.style` or `../project.mml` change , this is the 2 input files for detecting osm keys ### HOW to run: @@ -18,10 +18,19 @@ see more https://github.com/gravitystorm/openstreetmap-carto/issues/961 ### Known limitations * Only a subset of hstore `tags->` is parsed from the `../project.mml` -* Don't work the hstore ARRAY parsing: `tags @> ARRAY['building:level','building:color']` TODO * This code tested only on Ubuntu Linux * Check the result! +### Examples for parsing +* `tags @> 'capital=>yes'"]` +* `tags ? 'wetland'"` +* `tags->'wetland' ` +* `tags->'leaf_type'` +* `tags @> '"generator:source"=>wind'` +* `tags -> ARRAY['wheelchair',ramp:wheelchair']` +* `tags ?& ARRAY['wheelchair',ramp:wheelchair']` +* `tags ?| ARRAY['wheelchair',ramp:wheelchair']` + ### How to debug: The taginfo project_list file should contain a link to this repo ( /taginfo-project/taginfo-openstreetmap-carto.json ) * https://github.com/taginfo/taginfo-projects/blob/master/project_list.txt diff --git a/taginfo-project/generate-taginfo-project-file.py b/taginfo-project/generate-taginfo-project-file.py index dab0711855..80c64c7549 100755 --- a/taginfo-project/generate-taginfo-project-file.py +++ b/taginfo-project/generate-taginfo-project-file.py @@ -13,7 +13,9 @@ import re import json import yaml +import sys +# ------------------------- parameters ---------------------------- taginfo = { "data_format": 1, "project": { @@ -26,11 +28,29 @@ "tags": [] } +osm2pgsql_file = '../openstreetmap-carto.style' +cartocss_project_file = '../project.mml' +search_url = 'https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=' +# ------------------------------------------------------------------------------- + + + +def processOSMkeys(_ds_geometry,_osmtype,_tag): + key=_tag.split("'")[1].split("=")[0].replace('"','') + if key: + print("--:", _ds_geometry,"->", _osmtype, " key:", key) + if key not in allhstoretags: + k = [ _osmtype ] + allhstoretags[key]=k + elif _osmtype not in allhstoretags[key]: + allhstoretags[key].append(_osmtype) + return + # # Parsing openstreetmap-carto.style file # -with open('../openstreetmap-carto.style', 'r') as style: +with open( osm2pgsql_file , 'r') as style: for line in style: if line[0] == '#': continue @@ -57,7 +77,7 @@ "key": key, "object_types": object_types, "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q="+key + "doc_url": search_url+key }) @@ -66,16 +86,27 @@ # Parsing "project.mml" file for the HSTORE keys ( tags-> ) # -with open('../project.mml', 'r') as f: + +with open( cartocss_project_file , 'r') as f: newf = yaml.load(f.read()) f.closed -tags_b = re.compile(r"tags[^'^)]*'.+?'") +# ---------------------------------- Examples -------------------- +# tags @> 'capital=>yes'"] +# tags ? 'wetland'" +# tags->'wetland' +# tags->'leaf_type' +# tags @> '"generator:source"=>wind' +re_tags_b = re.compile(r"[^a-zA-Z0-9_]tags[^'^)^\[^\]]*'.+?'") + + +# ---------------------------------- Examples -------------------- +# tags -> ARRAY['wheelchair',ramp:wheelchair'] +# tags ?& ARRAY['wheelchair',ramp:wheelchair'] +# tags ?| ARRAY['wheelchair',ramp:wheelchair'] +re_tags_array = re.compile( r"[^a-zA-Z0-9_]tags\s*[@\?-][>&\|]\s*[aA][rR][rR][aA][yY]\[.+?\]" ) -# ["tags @> 'capital=>yes'"] -# ["tags ? 'wetland'", "tags->'wetland'", "tags->'leaf_type'"] -# (tags @> '"generator:source"=>wind' allhstoretags={} @@ -83,31 +114,45 @@ print( "########### processing Layer: ", layer["id"]," ###########" ) ds_geometry = layer.get("geometry") - osmtype = '' - if ds_geometry: - if ds_geometry=='point': - osmtype='node' - elif ds_geometry=='linestring': - osmtype='way' - elif ds_geometry=='polygon': - osmtype='area' ds_type = layer["Datasource"].get("type") if ds_type and ds_type == "postgis": ds_table = layer["Datasource"].get("table") if ds_table: - tags01 = tags_b.findall(ds_table) + + osmtype = '' + if ds_geometry: + if (ds_geometry=='point'): + osmtype='node' + elif ds_geometry=='linestring': + osmtype='way' + elif ds_geometry=='polygon': + osmtype='area' + else: + # If no Geometry type - we try to guess the type. + if 'planet_osm_point' in ds_table.lower(): + osmtype='node' + elif 'planet_osm_polygon' in ds_table.lower(): + osmtype='area' + elif 'planet_osm_line' in ds_table.lower(): + osmtype='way' + elif 'planet_osm_ways' in ds_table.lower(): + osmtype='way' + else: + print( ds_table.lower() ) + + + tags01 = re_tags_b.findall(ds_table) if tags01: print(tags01) for tag in tags01: - key=tag.split("'")[1].split("=")[0].replace('"','') - if key: - print("--:", ds_geometry,"->", osmtype, " key:", key) - if key not in allhstoretags: - k = [ osmtype ] - allhstoretags[key]=k - elif osmtype not in allhstoretags[key]: - allhstoretags[key].append(osmtype) + processOSMkeys(ds_geometry,osmtype,tag) + + tagsa = re_tags_array.findall(ds_table) + if tagsa: + for tags in tagsa: + for tag in tags.split(','): + processOSMkeys(ds_geometry,osmtype,tag) for k in allhstoretags: # add "relation" if "area" or "way" @@ -119,7 +164,7 @@ "key": k, "object_types": allhstoretags[k], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q="+k + "doc_url": search_url+k }) @@ -127,4 +172,3 @@ with open('taginfo-openstreetmap-carto.json', 'w') as outfile: json.dump(taginfo, outfile, indent=4) - \ No newline at end of file diff --git a/taginfo-project/taginfo-openstreetmap-carto.json b/taginfo-project/taginfo-openstreetmap-carto.json index f1acd4774f..fc1bed33ee 100644 --- a/taginfo-project/taginfo-openstreetmap-carto.json +++ b/taginfo-project/taginfo-openstreetmap-carto.json @@ -2,7 +2,7 @@ "data_format": 1, "project": { "name": "OpenStreetMap Carto keys", - "description": "OpenStreetMap.org mapnik style, in CartoCSS", + "description": "Default OpenStreetMap.org style using CartoCSS", "project_url": "https://github.com/gravitystorm/openstreetmap-carto", "contact_name": "openstreetmap-carto maintainers", "contact_email": "openstreetmap-carto (at) gravitystorm (dot) co (dot) uk" From e6fad5ce3485b5fdefdb03d22909ed7967e784f2 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 14:00:17 +0200 Subject: [PATCH 06/28] add ./generate-taginfo-project-file.py to .travis check --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index faa6345520..9dfc9266d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,3 +23,5 @@ script: - find symbols/ -name '*.svg' | xargs xmllint --noout # Check the Lua transforms - lua scripts/lua/test.lua + # Check the Taginfo-project + - cd ./taginfo-project && python3 ./generate-taginfo-project-file.py && cd .. From 6490f80e988f6c0b4c1af28fb71360df9cf93e43 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 14:44:31 +0200 Subject: [PATCH 07/28] add "pip install pyyaml" to .travis ( missing yaml ) --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9dfc9266d5..3a3dbf7f4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,5 +23,6 @@ script: - find symbols/ -name '*.svg' | xargs xmllint --noout # Check the Lua transforms - lua scripts/lua/test.lua - # Check the Taginfo-project + # Check the Taginfo-project + - pip install pyyaml - cd ./taginfo-project && python3 ./generate-taginfo-project-file.py && cd .. From 766db8608dd93fafb39d6f25fa616fe37657be69 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 14:50:02 +0200 Subject: [PATCH 08/28] fix PyYAML install --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3a3dbf7f4a..2efaef9fef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,5 +24,5 @@ script: # Check the Lua transforms - lua scripts/lua/test.lua # Check the Taginfo-project - - pip install pyyaml + - pip3 install PyYAML - cd ./taginfo-project && python3 ./generate-taginfo-project-file.py && cd .. From f5639270499bcfc1d207907a2688d93622210778 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 15:02:36 +0200 Subject: [PATCH 09/28] fix travis python --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2efaef9fef..47ced1596b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,11 @@ addons: - libxml2-utils env: - CARTO=0.18.0 MAPNIK='3.0.0 3.0.12' + +before_install: + - pyenv global system 3.6 install: + - pip3 install PyYAML - npm install carto@$CARTO - mkdir -p data/world_boundaries data/simplified-land-polygons-complete-3857 data/ne_110m_admin_0_boundary_lines_land data/land-polygons-split-3857 - touch data/simplified-land-polygons-complete-3857/simplified_land_polygons.shp data/ne_110m_admin_0_boundary_lines_land/ne_110m_admin_0_boundary_lines_land.shp data/land-polygons-split-3857/land_polygons.shp @@ -24,5 +28,4 @@ script: # Check the Lua transforms - lua scripts/lua/test.lua # Check the Taginfo-project - - pip3 install PyYAML - cd ./taginfo-project && python3 ./generate-taginfo-project-file.py && cd .. From 19c222ea8a14b444fee46152baa6307b0d1a1ab7 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 15:11:52 +0200 Subject: [PATCH 10/28] fix python travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47ced1596b..29e9bc5b43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: before_install: - pyenv global system 3.6 install: - - pip3 install PyYAML + - pip install PyYAML - npm install carto@$CARTO - mkdir -p data/world_boundaries data/simplified-land-polygons-complete-3857 data/ne_110m_admin_0_boundary_lines_land data/land-polygons-split-3857 - touch data/simplified-land-polygons-complete-3857/simplified_land_polygons.shp data/ne_110m_admin_0_boundary_lines_land/ne_110m_admin_0_boundary_lines_land.shp data/land-polygons-split-3857/land_polygons.shp @@ -28,4 +28,4 @@ script: # Check the Lua transforms - lua scripts/lua/test.lua # Check the Taginfo-project - - cd ./taginfo-project && python3 ./generate-taginfo-project-file.py && cd .. + - cd ./taginfo-project && ./generate-taginfo-project-file.py && cd .. From cb48f816c2f4c6fa14bfac1be6eeb69a8b63e2ad Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 15:22:01 +0200 Subject: [PATCH 11/28] fix python travis --- .travis.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29e9bc5b43..ac17e62f58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,11 @@ addons: packages: - lua5.1 - libxml2-utils + - python3 env: - CARTO=0.18.0 MAPNIK='3.0.0 3.0.12' - -before_install: - - pyenv global system 3.6 install: - - pip install PyYAML + - pip3 install PyYAML - npm install carto@$CARTO - mkdir -p data/world_boundaries data/simplified-land-polygons-complete-3857 data/ne_110m_admin_0_boundary_lines_land data/land-polygons-split-3857 - touch data/simplified-land-polygons-complete-3857/simplified_land_polygons.shp data/ne_110m_admin_0_boundary_lines_land/ne_110m_admin_0_boundary_lines_land.shp data/land-polygons-split-3857/land_polygons.shp @@ -28,4 +26,4 @@ script: # Check the Lua transforms - lua scripts/lua/test.lua # Check the Taginfo-project - - cd ./taginfo-project && ./generate-taginfo-project-file.py && cd .. + - cd ./taginfo-project && python3 ./generate-taginfo-project-file.py && cd .. From 93eeb751bde3878640f9678511dac905875fbf4f Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 15:24:38 +0200 Subject: [PATCH 12/28] fix travis python3 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index ac17e62f58..8405754409 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ addons: - python3 env: - CARTO=0.18.0 MAPNIK='3.0.0 3.0.12' +before_install: + - pyenv global system 3.6 # Workaround for travis-ci/issues/8363 install: - pip3 install PyYAML - npm install carto@$CARTO From 7cbf6a1843af713abb2e38789400e38e82d748cd Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 15:53:43 +0200 Subject: [PATCH 13/28] fix python3 travis --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8405754409..5181f2f290 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: node_js dist: trusty -sudo: false +sudo: required node_js: - "6" addons: @@ -8,24 +8,24 @@ addons: packages: - lua5.1 - libxml2-utils - - python3 env: - CARTO=0.18.0 MAPNIK='3.0.0 3.0.12' -before_install: - - pyenv global system 3.6 # Workaround for travis-ci/issues/8363 + - TRAVIS_PYTHON_VERSION=3.6 + install: - - pip3 install PyYAML + - sudo pip install -U PyYAML - npm install carto@$CARTO - mkdir -p data/world_boundaries data/simplified-land-polygons-complete-3857 data/ne_110m_admin_0_boundary_lines_land data/land-polygons-split-3857 - touch data/simplified-land-polygons-complete-3857/simplified_land_polygons.shp data/ne_110m_admin_0_boundary_lines_land/ne_110m_admin_0_boundary_lines_land.shp data/land-polygons-split-3857/land_polygons.shp script: # We're using pipes in the checks, so fail if any part fails - set -o pipefail + # Check the Taginfo-project + - cd ./taginfo-project && ./generate-taginfo-project-file.py && cd .. # Validate the MML against multiple Mapnik versions, and report its lines for debugging purposes - for m in $MAPNIK; do ./node_modules/carto/bin/carto -a $m project.mml | xmllint - | wc -l; done # Validate that the SVGs are valid XML - find symbols/ -name '*.svg' | xargs xmllint --noout # Check the Lua transforms - lua scripts/lua/test.lua - # Check the Taginfo-project - - cd ./taginfo-project && python3 ./generate-taginfo-project-file.py && cd .. + From 3544f6cf178e1c2c32ecd77c3acb2231fe76ef17 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 15:56:09 +0200 Subject: [PATCH 14/28] fix travis --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5181f2f290..3fb611acda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,7 @@ addons: - lua5.1 - libxml2-utils env: - - CARTO=0.18.0 MAPNIK='3.0.0 3.0.12' - - TRAVIS_PYTHON_VERSION=3.6 + - CARTO=0.18.0 MAPNIK='3.0.0 3.0.12' TRAVIS_PYTHON_VERSION=3.6 install: - sudo pip install -U PyYAML @@ -28,4 +27,3 @@ script: - find symbols/ -name '*.svg' | xargs xmllint --noout # Check the Lua transforms - lua scripts/lua/test.lua - From 0d0794deb5eb70ffd89ea73ee46b9ffc22601cfc Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 16:51:05 +0200 Subject: [PATCH 15/28] fix travis python3 --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3fb611acda..d9db925e1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,14 @@ addons: packages: - lua5.1 - libxml2-utils + - python3 + env: - CARTO=0.18.0 MAPNIK='3.0.0 3.0.12' TRAVIS_PYTHON_VERSION=3.6 +before_install: + - pyenv global system 3.6 # Workaround for travis-ci/issues/8363 + install: - sudo pip install -U PyYAML - npm install carto@$CARTO From 2960203c05d8e041bf20f401afb3198b6275eb00 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 31 Mar 2018 16:56:35 +0200 Subject: [PATCH 16/28] fix python3 yaml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d9db925e1e..077e2c633b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ before_install: - pyenv global system 3.6 # Workaround for travis-ci/issues/8363 install: - - sudo pip install -U PyYAML + - sudo apt-get install -y python3-yaml - npm install carto@$CARTO - mkdir -p data/world_boundaries data/simplified-land-polygons-complete-3857 data/ne_110m_admin_0_boundary_lines_land data/land-polygons-split-3857 - touch data/simplified-land-polygons-complete-3857/simplified_land_polygons.shp data/ne_110m_admin_0_boundary_lines_land/ne_110m_admin_0_boundary_lines_land.shp data/land-polygons-split-3857/land_polygons.shp From b59ab9adecb043b4ec021bf9dd0b51b91ec81b11 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 2 Apr 2018 16:32:31 +0200 Subject: [PATCH 17/28] fix typos, improve python code, add '-v" --- taginfo-project/README.md | 19 +-- .../generate-taginfo-project-file.py | 109 +++++++++--------- 2 files changed, 68 insertions(+), 60 deletions(-) diff --git a/taginfo-project/README.md b/taginfo-project/README.md index 097b407032..96d6834931 100644 --- a/taginfo-project/README.md +++ b/taginfo-project/README.md @@ -4,8 +4,12 @@ The taginfo database keeps the information which projects use which OSM keys and * site: https://taginfo.openstreetmap.org * wiki: https://wiki.openstreetmap.org/wiki/Taginfo -Now We can generate only a minimal info - about the used 'keys' -see more https://github.com/gravitystorm/openstreetmap-carto/issues/961 + +_"There are many projects using OSM tags in some way: editors, routers, maps, data extraction tools, ... Taginfo can get information from those projects about their use of OSM tags through 'project files'."_ +_"Taginfo will periodically (usually daily) poll the project file and add the information to its own database."_ see more: https://wiki.openstreetmap.org/wiki/Taginfo/Projects + + +Now We can generate only a minimal info - about the used 'keys' ( see more https://github.com/gravitystorm/openstreetmap-carto/issues/961 ) ### WHEN to run? * when the `../openstreetmap-carto.style` or `../project.mml` change , this is the 2 input files for detecting osm keys @@ -17,16 +21,17 @@ see more https://github.com/gravitystorm/openstreetmap-carto/issues/961 * the new: `taginfo-openstreetmap-carto.json` ( and some debug info in the screen! ) ### Known limitations -* Only a subset of hstore `tags->` is parsed from the `../project.mml` +* Only a small subset of [hstore operators](https://www.postgresql.org/docs/10/static/hstore.html#HSTORE-OP-TABLE) is parsed from the `../project.mml` +* Be carefull with the SQL comments or use as a simple hack: ` /* tags->'wetland' */ ` * This code tested only on Ubuntu Linux * Check the result! -### Examples for parsing -* `tags @> 'capital=>yes'"]` +### Valid examples for parsing * `tags ? 'wetland'"` -* `tags->'wetland' ` -* `tags->'leaf_type'` * `tags @> '"generator:source"=>wind'` +* `tags @> 'capital=>yes'"]` +* `tags -> 'leaf_type'` +* `tags -> 'wetland'` * `tags -> ARRAY['wheelchair',ramp:wheelchair']` * `tags ?& ARRAY['wheelchair',ramp:wheelchair']` * `tags ?| ARRAY['wheelchair',ramp:wheelchair']` diff --git a/taginfo-project/generate-taginfo-project-file.py b/taginfo-project/generate-taginfo-project-file.py index 80c64c7549..42d34b56fd 100755 --- a/taginfo-project/generate-taginfo-project-file.py +++ b/taginfo-project/generate-taginfo-project-file.py @@ -2,18 +2,23 @@ # ------------------------------------------------------------------------------- # -# This code generate a taginfo project list file (see more https://wiki.openstreetmap.org/wiki/Taginfo/Projects ) +# This code generates a taginfo project list file (see more https://wiki.openstreetmap.org/wiki/Taginfo/Projects ) # -### Attribution & Disclaimer: +### Attribution & Disclaimer: # This code is based on : # Paul Norman code : https://github.com/osmlab/osm2pgsql_taginfo # Sven Geggus code : https://github.com/giggls/openstreetmap-carto-de/blob/master/views_osmde/generate_taginfo.py # ------------------------------------------------------- -import re import json +import os +import re import yaml -import sys +import argparse +parser = argparse.ArgumentParser() +parser.add_argument("-v", "--verbose", help="Print debug info",action="store_true") +args = parser.parse_args() + # ------------------------- parameters ---------------------------- taginfo = { @@ -27,24 +32,40 @@ }, "tags": [] } +cwd = os.getcwd() -osm2pgsql_file = '../openstreetmap-carto.style' -cartocss_project_file = '../project.mml' +osm2pgsql_file = os.path.join(cwd, '..', 'openstreetmap-carto.style') +cartocss_project_file = os.path.join(cwd, '..', 'project.mml') search_url = 'https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=' -# ------------------------------------------------------------------------------- +# ---------------------------------- Examples -------------------- +# tags @> 'capital=>yes'"] +# tags ? 'wetland'" +# tags->'wetland' +# tags->'leaf_type' +# tags @> '"generator:source"=>wind' +re_tags_one = re.compile(r"[^a-zA-Z0-9_]tags[^'^)^\[^\]]*'.+?'") + + +# ---------------------------------- Examples -------------------- +# tags -> ARRAY['wheelchair',ramp:wheelchair'] +# tags ?& ARRAY['wheelchair',ramp:wheelchair'] +# tags ?| ARRAY['wheelchair',ramp:wheelchair'] +re_tags_array = re.compile( r"[^a-zA-Z0-9_]tags\s*[@\?-][>&\|]\s*[aA][rR][rR][aA][yY]\[.+?\]" ) +allhstoretags={} -def processOSMkeys(_ds_geometry,_osmtype,_tag): +def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): key=_tag.split("'")[1].split("=")[0].replace('"','') if key: - print("--:", _ds_geometry,"->", _osmtype, " key:", key) + if args.verbose: + print("--:", _ds_geometry,"->", _osmtype, " key:", key) if key not in allhstoretags: k = [ _osmtype ] allhstoretags[key]=k elif _osmtype not in allhstoretags[key]: allhstoretags[key].append(_osmtype) - return + return # @@ -64,14 +85,14 @@ def processOSMkeys(_ds_geometry,_osmtype,_tag): if 'node' in keyline[0]: object_types.append('node') if 'way' in keyline[0]: - object_types.append('way') + object_types.append('way') if 'polygon' in keyline[3]: object_types.append('area') if ('area' in object_types) or ('way' in object_types ): object_types.append('relation') - - if key not in ('z_order','way_area'): + + if key not in ('z_order','way_area'): taginfo["tags"].append( { "key": key, @@ -86,35 +107,16 @@ def processOSMkeys(_ds_geometry,_osmtype,_tag): # Parsing "project.mml" file for the HSTORE keys ( tags-> ) # - with open( cartocss_project_file , 'r') as f: newf = yaml.load(f.read()) f.closed - -# ---------------------------------- Examples -------------------- -# tags @> 'capital=>yes'"] -# tags ? 'wetland'" -# tags->'wetland' -# tags->'leaf_type' -# tags @> '"generator:source"=>wind' -re_tags_b = re.compile(r"[^a-zA-Z0-9_]tags[^'^)^\[^\]]*'.+?'") - - -# ---------------------------------- Examples -------------------- -# tags -> ARRAY['wheelchair',ramp:wheelchair'] -# tags ?& ARRAY['wheelchair',ramp:wheelchair'] -# tags ?| ARRAY['wheelchair',ramp:wheelchair'] -re_tags_array = re.compile( r"[^a-zA-Z0-9_]tags\s*[@\?-][>&\|]\s*[aA][rR][rR][aA][yY]\[.+?\]" ) - - -allhstoretags={} - for layer in newf["Layer"]: - print( "########### processing Layer: ", layer["id"]," ###########" ) + if args.verbose: + print( "########### processing Layer: ", layer["id"]," ###########" ) + _layer = layer["id"] ds_geometry = layer.get("geometry") - ds_type = layer["Datasource"].get("type") if ds_type and ds_type == "postgis": ds_table = layer["Datasource"].get("table") @@ -124,38 +126,39 @@ def processOSMkeys(_ds_geometry,_osmtype,_tag): if ds_geometry: if (ds_geometry=='point'): osmtype='node' - elif ds_geometry=='linestring': + elif ds_geometry=='linestring': osmtype='way' - elif ds_geometry=='polygon': + elif ds_geometry=='polygon': osmtype='area' else: # If no Geometry type - we try to guess the type. if 'planet_osm_point' in ds_table.lower(): - osmtype='node' + osmtype='node' elif 'planet_osm_polygon' in ds_table.lower(): - osmtype='area' + osmtype='area' elif 'planet_osm_line' in ds_table.lower(): osmtype='way' elif 'planet_osm_ways' in ds_table.lower(): osmtype='way' else: - print( ds_table.lower() ) - - - tags01 = re_tags_b.findall(ds_table) - if tags01: - print(tags01) - for tag in tags01: - processOSMkeys(ds_geometry,osmtype,tag) - - tagsa = re_tags_array.findall(ds_table) - if tagsa: - for tags in tagsa: + if args.verbose: + print( 'table Not found:', ds_table.lower() ) + + tags_one = re_tags_one.findall(ds_table) + if tags_one: + if args.verbose: + print(tags_one) + for tag in tags_one: + processOSMkeys(_layer,ds_geometry,osmtype,tag) + + tags_array = re_tags_array.findall(ds_table) + if tags_array: + for tags in tags_array: for tag in tags.split(','): - processOSMkeys(ds_geometry,osmtype,tag) + processOSMkeys(_layer,ds_geometry,osmtype,tag) for k in allhstoretags: - # add "relation" if "area" or "way" + # add "relation" if "area" or "way" if ('area' in allhstoretags[k]) or ('way' in allhstoretags[k]): allhstoretags[k].append("relation") From 74db3d3d672ae17e45984695f4b4a3d39e5b50ab Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 2 Apr 2018 16:49:54 +0200 Subject: [PATCH 18/28] add "git diff taginfo-openstreetmap-carto.json" to the travis check --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 077e2c633b..e26cdaaa5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,8 @@ script: # We're using pipes in the checks, so fail if any part fails - set -o pipefail # Check the Taginfo-project - - cd ./taginfo-project && ./generate-taginfo-project-file.py && cd .. + - cd ./taginfo-project && ./generate-taginfo-project-file.py && cd .. + - cd ./taginfo-project && git diff taginfo-openstreetmap-carto.json && cd .. # Validate the MML against multiple Mapnik versions, and report its lines for debugging purposes - for m in $MAPNIK; do ./node_modules/carto/bin/carto -a $m project.mml | xmllint - | wc -l; done # Validate that the SVGs are valid XML From fdd5694c9af5b3cffb6e0c0ec173f344ee9fd833 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 2 Apr 2018 18:22:29 +0200 Subject: [PATCH 19/28] sort 'taginfo-openstreetmap-carto.json' by osm key --- .../generate-taginfo-project-file.py | 8 +- .../taginfo-openstreetmap-carto.json | 416 +++++++++--------- 2 files changed, 213 insertions(+), 211 deletions(-) diff --git a/taginfo-project/generate-taginfo-project-file.py b/taginfo-project/generate-taginfo-project-file.py index 42d34b56fd..1bc86ca366 100755 --- a/taginfo-project/generate-taginfo-project-file.py +++ b/taginfo-project/generate-taginfo-project-file.py @@ -10,11 +10,12 @@ # Sven Geggus code : https://github.com/giggls/openstreetmap-carto-de/blob/master/views_osmde/generate_taginfo.py # ------------------------------------------------------- +import argparse import json import os import re import yaml -import argparse + parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", help="Print debug info",action="store_true") args = parser.parse_args() @@ -173,5 +174,6 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): with open('taginfo-openstreetmap-carto.json', 'w') as outfile: - json.dump(taginfo, outfile, indent=4) - + taginfo_sorted=taginfo + taginfo_sorted["tags"]= sorted(taginfo["tags"],key= lambda k: k['key'] ) + json.dump(taginfo_sorted, outfile, indent=4) diff --git a/taginfo-project/taginfo-openstreetmap-carto.json b/taginfo-project/taginfo-openstreetmap-carto.json index fc1bed33ee..72032bbb3c 100644 --- a/taginfo-project/taginfo-openstreetmap-carto.json +++ b/taginfo-project/taginfo-openstreetmap-carto.json @@ -47,6 +47,14 @@ "description": "Used in the osm2pgsql database backend, see more in the github repo", "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=addr:interpolation" }, + { + "key": "addr:unit", + "object_types": [ + "node" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=addr:unit" + }, { "key": "admin_level", "object_types": [ @@ -57,6 +65,16 @@ "description": "Used in the osm2pgsql database backend, see more in the github repo", "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=admin_level" }, + { + "key": "advertising", + "object_types": [ + "area", + "node", + "relation" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=advertising" + }, { "key": "aerialway", "object_types": [ @@ -109,23 +127,23 @@ "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=bicycle" }, { - "key": "bridge", + "key": "boundary", "object_types": [ + "node", "way", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=bridge" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=boundary" }, { - "key": "boundary", + "key": "bridge", "object_types": [ - "node", "way", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=boundary" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=bridge" }, { "key": "building", @@ -138,6 +156,14 @@ "description": "Used in the osm2pgsql database backend, see more in the github repo", "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=building" }, + { + "key": "capital", + "object_types": [ + "node" + ], + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=capital" + }, { "key": "construction", "object_types": [ @@ -157,109 +183,101 @@ "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=covered" }, { - "key": "foot", + "key": "denomination", "object_types": [ - "way", + "area", + "node", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=foot" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=denomination" }, { - "key": "highway", + "key": "ele", "object_types": [ "node", - "way", + "area", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=highway" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ele" }, { - "key": "historic", + "key": "emergency", "object_types": [ - "node", - "way", - "area", - "relation" + "node" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=historic" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=emergency" }, { - "key": "horse", + "key": "entrance", "object_types": [ - "way", - "relation" + "node" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=horse" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=entrance" }, { - "key": "junction", + "key": "foot", "object_types": [ - "node", "way", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=junction" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=foot" }, { - "key": "landuse", + "key": "ford", "object_types": [ - "node", "way", - "area", + "node", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=landuse" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ford" }, { - "key": "layer", + "key": "generator:source", "object_types": [ + "area", "node", - "way", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=layer" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=generator:source" }, { - "key": "leisure", + "key": "height", "object_types": [ - "node", - "way", "area", + "node", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=leisure" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=height" }, { - "key": "lock", + "key": "highspeed", "object_types": [ - "node", "way", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=lock" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=highspeed" }, { - "key": "man_made", + "key": "highway", "object_types": [ "node", "way", - "area", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=man_made" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=highway" }, { - "key": "military", + "key": "historic", "object_types": [ "node", "way", @@ -267,160 +285,160 @@ "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=military" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=historic" }, { - "key": "name", + "key": "horse", "object_types": [ - "node", "way", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=name" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=horse" }, { - "key": "natural", + "key": "iata", "object_types": [ - "node", - "way", "area", + "node", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=natural" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=iata" }, { - "key": "oneway", + "key": "icao", "object_types": [ + "area", "node", - "way", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=oneway" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=icao" }, { - "key": "place", + "key": "indoor", "object_types": [ - "node", - "way", - "area", - "relation" + "node" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=place" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=indoor" }, { - "key": "power", + "key": "intermittent", "object_types": [ - "node", "way", "area", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=power" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=intermittent" }, { - "key": "railway", + "key": "junction", "object_types": [ "node", "way", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=railway" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=junction" }, { - "key": "ref", + "key": "landuse", "object_types": [ "node", "way", + "area", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ref" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=landuse" }, { - "key": "religion", + "key": "layer", "object_types": [ "node", "way", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=religion" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=layer" }, { - "key": "route", + "key": "leaf_type", "object_types": [ - "way", + "area", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=route" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=leaf_type" }, { - "key": "service", + "key": "leisure", "object_types": [ + "node", "way", + "area", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=service" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=leisure" }, { - "key": "shop", + "key": "lock", "object_types": [ "node", "way", - "area", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=shop" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=lock" }, { - "key": "surface", + "key": "man_made", "object_types": [ + "node", "way", + "area", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=surface" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=man_made" }, { - "key": "tourism", + "key": "memorial", "object_types": [ - "node", - "way", "area", + "node", "relation" ], - "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tourism" + "description": "Used as a hstore tags-> in the database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=memorial" }, { - "key": "tracktype", + "key": "military", "object_types": [ + "node", "way", + "area", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tracktype" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=military" }, { - "key": "tunnel", + "key": "name", "object_types": [ + "node", "way", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tunnel" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=name" }, { - "key": "water", + "key": "natural", "object_types": [ "node", "way", @@ -428,56 +446,68 @@ "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=water" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=natural" }, { - "key": "waterway", + "key": "oneway", "object_types": [ "node", "way", - "area", "relation" ], "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=waterway" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=oneway" }, { - "key": "wetland", + "key": "operator", "object_types": [ "area", + "way", + "node", "relation" ], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=wetland" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=operator" }, { - "key": "intermittent", + "key": "place", "object_types": [ + "node", "way", "area", "relation" ], + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=place" + }, + { + "key": "population", + "object_types": [ + "node" + ], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=intermittent" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=population" }, { - "key": "seasonal", + "key": "power", "object_types": [ + "node", "way", "area", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=seasonal" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=power" }, { - "key": "leaf_type", + "key": "power_source", "object_types": [ "area", + "node", "relation" ], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=leaf_type" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=power_source" }, { "key": "public_transport", @@ -489,126 +519,103 @@ "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=public_transport" }, { - "key": "entrance", - "object_types": [ - "node" - ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=entrance" - }, - { - "key": "indoor", - "object_types": [ - "node" - ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=indoor" - }, - { - "key": "population", - "object_types": [ - "node" - ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=population" - }, - { - "key": "capital", + "key": "railway", "object_types": [ - "node" + "node", + "way", + "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=capital" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=railway" }, { - "key": "advertising", + "key": "recycling_type", "object_types": [ "area", "node", "relation" ], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=advertising" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=recycling_type" }, { - "key": "memorial", + "key": "ref", "object_types": [ - "area", "node", + "way", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=memorial" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ref" }, { - "key": "denomination", + "key": "religion", "object_types": [ - "area", "node", + "way", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=denomination" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=religion" }, { - "key": "generator:source", + "key": "route", "object_types": [ - "area", - "node", + "way", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=generator:source" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=route" }, { - "key": "height", + "key": "seasonal", "object_types": [ + "way", "area", - "node", "relation" ], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=height" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=seasonal" }, { - "key": "power_source", + "key": "service", "object_types": [ - "area", - "node", + "way", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=power_source" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=service" }, { - "key": "icao", + "key": "shop", "object_types": [ - "area", "node", + "way", + "area", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=icao" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=shop" }, { - "key": "iata", + "key": "surface", "object_types": [ - "area", - "node", + "way", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=iata" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=surface" }, { - "key": "recycling_type", + "key": "tourism", "object_types": [ - "area", "node", + "way", + "area", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=recycling_type" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tourism" }, { "key": "tower:construction", @@ -631,69 +638,62 @@ "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tower:type" }, { - "key": "ford", + "key": "tracktype", "object_types": [ "way", - "node", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ford" - }, - { - "key": "emergency", - "object_types": [ - "node" - ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=emergency" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tracktype" }, { - "key": "ele", + "key": "tunnel", "object_types": [ - "node", - "area", + "way", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=ele" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=tunnel" }, { - "key": "highspeed", + "key": "usage", "object_types": [ "way", "relation" ], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=highspeed" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=usage" }, { - "key": "usage", + "key": "water", "object_types": [ + "node", "way", + "area", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=usage" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=water" }, { - "key": "operator", + "key": "waterway", "object_types": [ - "area", - "way", "node", + "way", + "area", "relation" ], - "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=operator" + "description": "Used in the osm2pgsql database backend, see more in the github repo", + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=waterway" }, { - "key": "addr:unit", + "key": "wetland", "object_types": [ - "node" + "area", + "relation" ], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=addr:unit" + "doc_url": "https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=wetland" } ] } \ No newline at end of file From 79ef7f54b217b3ff4017f8c45f797e837576b4eb Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 3 Apr 2018 00:00:08 +0200 Subject: [PATCH 20/28] change suggested link to " https://taginfo.openstreetmap.org/projects/openstreetmap-carto" --- taginfo-project/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taginfo-project/README.md b/taginfo-project/README.md index 96d6834931..88b28443c0 100644 --- a/taginfo-project/README.md +++ b/taginfo-project/README.md @@ -41,10 +41,10 @@ The taginfo project_list file should contain a link to this repo ( /taginfo-proj * https://github.com/taginfo/taginfo-projects/blob/master/project_list.txt the expected line: -* openstreetmap_carto https://raw.githubusercontent.com/gravitystorm/openstreetmap-carto/master/taginfo-project/taginfo-openstreetmap-carto.json +* openstreetmap-carto https://raw.githubusercontent.com/gravitystorm/openstreetmap-carto/master/taginfo-project/taginfo-openstreetmap-carto.json After the daily refresh the project info should be find here: -* https://taginfo.openstreetmap.org/projects/openstreetmap_carto +* https://taginfo.openstreetmap.org/projects/openstreetmap-carto ### Disclaimer and Attribution : This code is based on * Paul Norman code : https://github.com/osmlab/osm2pgsql_taginfo From fec1f49d63b24c3ac2bfe3cc253d3d7b891ab547 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 3 Apr 2018 16:45:58 +0200 Subject: [PATCH 21/28] move taginfo files to the new place --- {taginfo-project => scripts/taginfo-project}/README.md | 0 .../taginfo-project}/generate-taginfo-project-file.py | 0 ...o-openstreetmap-carto.json => taginfo-openstreetmap-carto.json | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {taginfo-project => scripts/taginfo-project}/README.md (100%) rename {taginfo-project => scripts/taginfo-project}/generate-taginfo-project-file.py (100%) rename taginfo-project/taginfo-openstreetmap-carto.json => taginfo-openstreetmap-carto.json (100%) diff --git a/taginfo-project/README.md b/scripts/taginfo-project/README.md similarity index 100% rename from taginfo-project/README.md rename to scripts/taginfo-project/README.md diff --git a/taginfo-project/generate-taginfo-project-file.py b/scripts/taginfo-project/generate-taginfo-project-file.py similarity index 100% rename from taginfo-project/generate-taginfo-project-file.py rename to scripts/taginfo-project/generate-taginfo-project-file.py diff --git a/taginfo-project/taginfo-openstreetmap-carto.json b/taginfo-openstreetmap-carto.json similarity index 100% rename from taginfo-project/taginfo-openstreetmap-carto.json rename to taginfo-openstreetmap-carto.json From 6a5fb1bfb55cc934cf5ebb6aa22960b420b29fff Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 3 Apr 2018 16:59:40 +0200 Subject: [PATCH 22/28] fix taginfo-project scripts path --- .travis.yml | 6 +++--- scripts/taginfo-project/README.md | 10 +++++----- .../taginfo-project/generate-taginfo-project-file.py | 10 +++++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index e26cdaaa5a..5457d02733 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ env: before_install: - pyenv global system 3.6 # Workaround for travis-ci/issues/8363 - + install: - sudo apt-get install -y python3-yaml - npm install carto@$CARTO @@ -25,8 +25,8 @@ script: # We're using pipes in the checks, so fail if any part fails - set -o pipefail # Check the Taginfo-project - - cd ./taginfo-project && ./generate-taginfo-project-file.py && cd .. - - cd ./taginfo-project && git diff taginfo-openstreetmap-carto.json && cd .. + - cd ./scripts/taginfo-project && ./generate-taginfo-project-file.py && cd ../.. + - git diff taginfo-openstreetmap-carto.json # Validate the MML against multiple Mapnik versions, and report its lines for debugging purposes - for m in $MAPNIK; do ./node_modules/carto/bin/carto -a $m project.mml | xmllint - | wc -l; done # Validate that the SVGs are valid XML diff --git a/scripts/taginfo-project/README.md b/scripts/taginfo-project/README.md index 88b28443c0..89ad0881ee 100644 --- a/scripts/taginfo-project/README.md +++ b/scripts/taginfo-project/README.md @@ -12,16 +12,16 @@ _"Taginfo will periodically (usually daily) poll the project file and add the in Now We can generate only a minimal info - about the used 'keys' ( see more https://github.com/gravitystorm/openstreetmap-carto/issues/961 ) ### WHEN to run? -* when the `../openstreetmap-carto.style` or `../project.mml` change , this is the 2 input files for detecting osm keys +* when the `../,,/openstreetmap-carto.style` or `../../project.mml` change , this is the 2 input files for detecting osm keys ### HOW to run: * from this directory: `python3 ./generate-taginfo-project-file.py` ### RESULT: -* the new: `taginfo-openstreetmap-carto.json` ( and some debug info in the screen! ) +* the new: `../../taginfo-openstreetmap-carto.json` ### Known limitations -* Only a small subset of [hstore operators](https://www.postgresql.org/docs/10/static/hstore.html#HSTORE-OP-TABLE) is parsed from the `../project.mml` +* Only a small subset of [hstore operators](https://www.postgresql.org/docs/10/static/hstore.html#HSTORE-OP-TABLE) is parsed from the `../../project.mml` * Be carefull with the SQL comments or use as a simple hack: ` /* tags->'wetland' */ ` * This code tested only on Ubuntu Linux * Check the result! @@ -37,11 +37,11 @@ Now We can generate only a minimal info - about the used 'keys' ( see more http * `tags ?| ARRAY['wheelchair',ramp:wheelchair']` ### How to debug: -The taginfo project_list file should contain a link to this repo ( /taginfo-project/taginfo-openstreetmap-carto.json ) +The taginfo project_list file should contain a link to this repo ( `taginfo-openstreetmap-carto.json` ) * https://github.com/taginfo/taginfo-projects/blob/master/project_list.txt the expected line: -* openstreetmap-carto https://raw.githubusercontent.com/gravitystorm/openstreetmap-carto/master/taginfo-project/taginfo-openstreetmap-carto.json +* openstreetmap-carto https://raw.githubusercontent.com/gravitystorm/openstreetmap-carto/master/taginfo-openstreetmap-carto.json After the daily refresh the project info should be find here: * https://taginfo.openstreetmap.org/projects/openstreetmap-carto diff --git a/scripts/taginfo-project/generate-taginfo-project-file.py b/scripts/taginfo-project/generate-taginfo-project-file.py index 1bc86ca366..7144fe4406 100755 --- a/scripts/taginfo-project/generate-taginfo-project-file.py +++ b/scripts/taginfo-project/generate-taginfo-project-file.py @@ -35,10 +35,14 @@ } cwd = os.getcwd() -osm2pgsql_file = os.path.join(cwd, '..', 'openstreetmap-carto.style') -cartocss_project_file = os.path.join(cwd, '..', 'project.mml') +osm2pgsql_file = os.path.join(cwd, '..','..', 'openstreetmap-carto.style') +cartocss_project_file = os.path.join(cwd, '..','..', 'project.mml') search_url = 'https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=' +# output +taginfo_project_file = os.path.join(cwd, '..','..', 'taginfo-openstreetmap-carto.json') + + # ---------------------------------- Examples -------------------- # tags @> 'capital=>yes'"] # tags ? 'wetland'" @@ -173,7 +177,7 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): -with open('taginfo-openstreetmap-carto.json', 'w') as outfile: +with open(taginfo_project_file, 'w') as outfile: taginfo_sorted=taginfo taginfo_sorted["tags"]= sorted(taginfo["tags"],key= lambda k: k['key'] ) json.dump(taginfo_sorted, outfile, indent=4) From b39eace761bf813dd626efd12e65299e479992e5 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 22 Apr 2018 13:04:10 +0200 Subject: [PATCH 23/28] using re.IGNORECASE --- scripts/taginfo-project/generate-taginfo-project-file.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/taginfo-project/generate-taginfo-project-file.py b/scripts/taginfo-project/generate-taginfo-project-file.py index 7144fe4406..7fe59fe830 100755 --- a/scripts/taginfo-project/generate-taginfo-project-file.py +++ b/scripts/taginfo-project/generate-taginfo-project-file.py @@ -49,14 +49,14 @@ # tags->'wetland' # tags->'leaf_type' # tags @> '"generator:source"=>wind' -re_tags_one = re.compile(r"[^a-zA-Z0-9_]tags[^'^)^\[^\]]*'.+?'") +re_tags_one = re.compile(r"[^a-z0-9_]tags[^'^)^\[^\]]*'.+?'") # calling with re.IGNORECASE # ---------------------------------- Examples -------------------- # tags -> ARRAY['wheelchair',ramp:wheelchair'] # tags ?& ARRAY['wheelchair',ramp:wheelchair'] # tags ?| ARRAY['wheelchair',ramp:wheelchair'] -re_tags_array = re.compile( r"[^a-zA-Z0-9_]tags\s*[@\?-][>&\|]\s*[aA][rR][rR][aA][yY]\[.+?\]" ) +re_tags_array = re.compile( r"[^a-z0-9_]tags\s*[@\?-][>&\|]\s*array\[.+?\]" ) # calling with re.IGNORECASE allhstoretags={} @@ -149,14 +149,14 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): if args.verbose: print( 'table Not found:', ds_table.lower() ) - tags_one = re_tags_one.findall(ds_table) + tags_one = re_tags_one.findall(ds_table,re.IGNORECASE) if tags_one: if args.verbose: print(tags_one) for tag in tags_one: processOSMkeys(_layer,ds_geometry,osmtype,tag) - tags_array = re_tags_array.findall(ds_table) + tags_array = re_tags_array.findall(ds_table,re.IGNORECASE) if tags_array: for tags in tags_array: for tag in tags.split(','): From 461b3dbe0830a85244e65eca2d9759ba5743e01d Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 22 Apr 2018 13:13:56 +0200 Subject: [PATCH 24/28] remove unnecessary close --- scripts/taginfo-project/generate-taginfo-project-file.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/taginfo-project/generate-taginfo-project-file.py b/scripts/taginfo-project/generate-taginfo-project-file.py index 7fe59fe830..badf8a01d7 100755 --- a/scripts/taginfo-project/generate-taginfo-project-file.py +++ b/scripts/taginfo-project/generate-taginfo-project-file.py @@ -114,7 +114,6 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): with open( cartocss_project_file , 'r') as f: newf = yaml.load(f.read()) -f.closed for layer in newf["Layer"]: if args.verbose: @@ -143,7 +142,7 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): osmtype='area' elif 'planet_osm_line' in ds_table.lower(): osmtype='way' - elif 'planet_osm_ways' in ds_table.lower(): + elif 'planet_osm_roads' in ds_table.lower(): osmtype='way' else: if args.verbose: From 8df146a530bc2ceb3b0db267729df686faea50f2 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 22 Apr 2018 13:20:20 +0200 Subject: [PATCH 25/28] remoove duplicated example --- scripts/taginfo-project/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/taginfo-project/README.md b/scripts/taginfo-project/README.md index 89ad0881ee..92ec7d1838 100644 --- a/scripts/taginfo-project/README.md +++ b/scripts/taginfo-project/README.md @@ -31,7 +31,6 @@ Now We can generate only a minimal info - about the used 'keys' ( see more http * `tags @> '"generator:source"=>wind'` * `tags @> 'capital=>yes'"]` * `tags -> 'leaf_type'` -* `tags -> 'wetland'` * `tags -> ARRAY['wheelchair',ramp:wheelchair']` * `tags ?& ARRAY['wheelchair',ramp:wheelchair']` * `tags ?| ARRAY['wheelchair',ramp:wheelchair']` From 07ea490aa5f6bb6166a4d3f40de7ea5094245c80 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 22 Apr 2018 19:22:22 +0200 Subject: [PATCH 26/28] taginfo: improve documentation / add extra parameters / refactor .travis --- .travis.yml | 6 +- scripts/taginfo-project/README.md | 48 +++++++++++- .../generate-taginfo-project-file.py | 73 ++++++++++++------- 3 files changed, 95 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5457d02733..e93ee8effa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,9 @@ install: script: # We're using pipes in the checks, so fail if any part fails - set -o pipefail - # Check the Taginfo-project - - cd ./scripts/taginfo-project && ./generate-taginfo-project-file.py && cd ../.. - - git diff taginfo-openstreetmap-carto.json + # generate the Taginfo-project file and show the differences + - ./scripts/taginfo-project/generate-taginfo-project-file.py -o ./new_carto_taginfo.json -v + - diff ./taginfo-openstreetmap-carto.json ./new_carto_taginfo.json # Validate the MML against multiple Mapnik versions, and report its lines for debugging purposes - for m in $MAPNIK; do ./node_modules/carto/bin/carto -a $m project.mml | xmllint - | wc -l; done # Validate that the SVGs are valid XML diff --git a/scripts/taginfo-project/README.md b/scripts/taginfo-project/README.md index 92ec7d1838..80e7215c6b 100644 --- a/scripts/taginfo-project/README.md +++ b/scripts/taginfo-project/README.md @@ -12,13 +12,15 @@ _"Taginfo will periodically (usually daily) poll the project file and add the in Now We can generate only a minimal info - about the used 'keys' ( see more https://github.com/gravitystorm/openstreetmap-carto/issues/961 ) ### WHEN to run? -* when the `../,,/openstreetmap-carto.style` or `../../project.mml` change , this is the 2 input files for detecting osm keys +* when the `openstreetmap-carto.style` or `project.mml` change, this is the 2 input files for detecting osm keys ### HOW to run: -* from this directory: `python3 ./generate-taginfo-project-file.py` +* from the project directory: `python3 scripts/taginfo-project/generate-taginfo-project-file.py` +* output: + * `taginfo-openstreetmap-carto.json` file +### With verbose mode and different file output +* `python3 ./scripts/taginfo-project/generate-taginfo-project-file.py -o ./new_carto_taginfo.json -v` -### RESULT: -* the new: `../../taginfo-openstreetmap-carto.json` ### Known limitations * Only a small subset of [hstore operators](https://www.postgresql.org/docs/10/static/hstore.html#HSTORE-OP-TABLE) is parsed from the `../../project.mml` @@ -45,6 +47,44 @@ the expected line: After the daily refresh the project info should be find here: * https://taginfo.openstreetmap.org/projects/openstreetmap-carto +### Command line parameters +``` +$ python3 scripts/taginfo-project/generate-taginfo-project-file.py -h + +usage: generate-taginfo-project-file.py [-h] [-v] + [--osm2pgsql_file OSM2PGSQL_FILE] + [--cartocss_project_file CARTOCSS_PROJECT_FILE] + [--taginfo_project_file TAGINFO_PROJECT_FILE] + [--project_name PROJECT_NAME] + [--project_description PROJECT_DESCRIPTION] + [--project_url PROJECT_URL] + [--contact_name CONTACT_NAME] + [--contact_email CONTACT_EMAIL] + [--search_url SEARCH_URL] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Print debug info + --osm2pgsql_file OSM2PGSQL_FILE + osm2pgsql config file + --cartocss_project_file CARTOCSS_PROJECT_FILE + project cartocss yml file + --taginfo_project_file TAGINFO_PROJECT_FILE, -o TAGINFO_PROJECT_FILE + output taginfo json file + --project_name PROJECT_NAME + taginfo project name + --project_description PROJECT_DESCRIPTION + taginfo project description + --project_url PROJECT_URL + taginfo project url + --contact_name CONTACT_NAME + taginfo project - contact name + --contact_email CONTACT_EMAIL + taginfo project - contact_email + --search_url SEARCH_URL + taginfo project - search_url for every keys +``` + ### Disclaimer and Attribution : This code is based on * Paul Norman code : https://github.com/osmlab/osm2pgsql_taginfo * Sven Geggus code : https://github.com/giggls/openstreetmap-carto-de/blob/master/views_osmde/generate_taginfo.py diff --git a/scripts/taginfo-project/generate-taginfo-project-file.py b/scripts/taginfo-project/generate-taginfo-project-file.py index badf8a01d7..97e2d8c236 100755 --- a/scripts/taginfo-project/generate-taginfo-project-file.py +++ b/scripts/taginfo-project/generate-taginfo-project-file.py @@ -17,36 +17,39 @@ import yaml parser = argparse.ArgumentParser() -parser.add_argument("-v", "--verbose", help="Print debug info",action="store_true") -args = parser.parse_args() +parser.add_argument("-v", "--verbose", help="Print debug info",action="store_true") + +parser.add_argument("--osm2pgsql_file", help="osm2pgsql config file", default="./openstreetmap-carto.style") +parser.add_argument("--cartocss_project_file", help="project cartocss yml file",default="./project.mml") +parser.add_argument("--taginfo_project_file", "-o", help="output taginfo json file ",default="./taginfo-openstreetmap-carto.json") + +parser.add_argument("--project_name", help="taginfo project name " ,default="OpenStreetMap Carto keys") +parser.add_argument("--project_description",help="taginfo project description" ,default="Default OpenStreetMap.org style using CartoCSS") +parser.add_argument("--project_url", help="taginfo project url " ,default="https://github.com/gravitystorm/openstreetmap-carto") +parser.add_argument("--contact_name", help="taginfo project - contact name ",default="openstreetmap-carto maintainers") +parser.add_argument("--contact_email", help="taginfo project - contact_email",default="openstreetmap-carto (at) gravitystorm (dot) co (dot) uk") + +parser.add_argument("--search_url",help="taginfo project - search_url for every keys",default='https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=') +args = parser.parse_args() # ------------------------- parameters ---------------------------- taginfo = { "data_format": 1, "project": { - "name": "OpenStreetMap Carto keys", - "description": "Default OpenStreetMap.org style using CartoCSS", - "project_url": "https://github.com/gravitystorm/openstreetmap-carto", - "contact_name": "openstreetmap-carto maintainers", - "contact_email": "openstreetmap-carto (at) gravitystorm (dot) co (dot) uk" + "name": args.project_name, + "description": args.project_description, + "project_url": args.project_url, + "contact_name": args.contact_name, + "contact_email":args.contact_email }, "tags": [] } -cwd = os.getcwd() - -osm2pgsql_file = os.path.join(cwd, '..','..', 'openstreetmap-carto.style') -cartocss_project_file = os.path.join(cwd, '..','..', 'project.mml') -search_url = 'https://github.com/gravitystorm/openstreetmap-carto/search?utf8=%E2%9C%93&q=' - -# output -taginfo_project_file = os.path.join(cwd, '..','..', 'taginfo-openstreetmap-carto.json') # ---------------------------------- Examples -------------------- # tags @> 'capital=>yes'"] # tags ? 'wetland'" -# tags->'wetland' # tags->'leaf_type' # tags @> '"generator:source"=>wind' re_tags_one = re.compile(r"[^a-z0-9_]tags[^'^)^\[^\]]*'.+?'") # calling with re.IGNORECASE @@ -60,11 +63,31 @@ allhstoretags={} -def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): - key=_tag.split("'")[1].split("=")[0].replace('"','') + + +def processOSMkeys(_layer, _ds_geometry, _osmtype, _tag): + """Processing sql codefragment(_tag) to allhstoretags[key] + + example input: + _layer = amenity-points + _ds_geometry = point + _osmtype = node + _tag = (tags @> '"generator:source"=>wind' + + should append/create + allhstoretags['generator:source']= [ node ] + + """ + + key=_tag.split("'")[1].split("=")[0].replace('"','') + # key=generator:source + if key: if args.verbose: - print("--:", _ds_geometry,"->", _osmtype, " key:", key) + print("--:[", _layer , _ds_geometry, _osmtype, '] parse:', _tag, " ----> key:", key) + # --:[ amenity-points point node ] parse: (tags @> '"generator:source"=>wind' ----> key: generator:source + + # based on example add/append: allhstoretags['generator:source']= [ node ] if key not in allhstoretags: k = [ _osmtype ] allhstoretags[key]=k @@ -76,7 +99,7 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): # # Parsing openstreetmap-carto.style file # -with open( osm2pgsql_file , 'r') as style: +with open( args.osm2pgsql_file , 'r') as style: for line in style: if line[0] == '#': continue @@ -103,7 +126,7 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): "key": key, "object_types": object_types, "description": "Used in the osm2pgsql database backend, see more in the github repo", - "doc_url": search_url+key + "doc_url": args.search_url+key }) @@ -112,7 +135,7 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): # Parsing "project.mml" file for the HSTORE keys ( tags-> ) # -with open( cartocss_project_file , 'r') as f: +with open( args.cartocss_project_file , 'r') as f: newf = yaml.load(f.read()) for layer in newf["Layer"]: @@ -171,12 +194,12 @@ def processOSMkeys(_layer, _ds_geometry, _osmtype,_tag): "key": k, "object_types": allhstoretags[k], "description": "Used as a hstore tags-> in the database backend, see more in the github repo", - "doc_url": search_url+k + "doc_url": args.search_url+k }) - -with open(taginfo_project_file, 'w') as outfile: +# write the json output +with open(args.taginfo_project_file, 'w') as outfile: taginfo_sorted=taginfo taginfo_sorted["tags"]= sorted(taginfo["tags"],key= lambda k: k['key'] ) json.dump(taginfo_sorted, outfile, indent=4) From 82b272f3f8e496979e15d02a514cd35879ab0962 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 22 Apr 2018 19:33:13 +0200 Subject: [PATCH 27/28] taginfo: fix .travis diff --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e93ee8effa..ce46215cf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ script: - set -o pipefail # generate the Taginfo-project file and show the differences - ./scripts/taginfo-project/generate-taginfo-project-file.py -o ./new_carto_taginfo.json -v - - diff ./taginfo-openstreetmap-carto.json ./new_carto_taginfo.json + - diff ./taginfo-openstreetmap-carto.json ./new_carto_taginfo.json | true # Validate the MML against multiple Mapnik versions, and report its lines for debugging purposes - for m in $MAPNIK; do ./node_modules/carto/bin/carto -a $m project.mml | xmllint - | wc -l; done # Validate that the SVGs are valid XML From b4476ac1d9bc188bbb07cbca680e37189aeabaf6 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 22 Apr 2018 19:43:15 +0200 Subject: [PATCH 28/28] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ce46215cf2..ed559f0c0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ script: - set -o pipefail # generate the Taginfo-project file and show the differences - ./scripts/taginfo-project/generate-taginfo-project-file.py -o ./new_carto_taginfo.json -v - - diff ./taginfo-openstreetmap-carto.json ./new_carto_taginfo.json | true + - diff ./taginfo-openstreetmap-carto.json ./new_carto_taginfo.json || true # Validate the MML against multiple Mapnik versions, and report its lines for debugging purposes - for m in $MAPNIK; do ./node_modules/carto/bin/carto -a $m project.mml | xmllint - | wc -l; done # Validate that the SVGs are valid XML