From 6f2c48b756a2a2183b2802ca917e925b834052fa Mon Sep 17 00:00:00 2001 From: eli knaap Date: Tue, 30 Jun 2020 09:18:26 -0700 Subject: [PATCH 01/25] update project_gdf --- osmnet/load.py | 528 ++++++++++++++++++++++++++++--------------------- 1 file changed, 307 insertions(+), 221 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index 66a0335..8c431a0 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -45,19 +45,23 @@ def osm_filter(network_type): # passenger vehicles both private and public # roads. Filter out un-drivable roads and service roads tagged as parking, # driveway, or emergency-access - filters['drive'] = ('["highway"!~"cycleway|footway|path|pedestrian|steps' - '|track|proposed|construction|bridleway|abandoned' - '|platform|raceway|service"]' - '["motor_vehicle"!~"no"]["motorcar"!~"no"]' - '["service"!~"parking|parking_aisle|driveway' - '|emergency_access"]') + filters["drive"] = ( + '["highway"!~"cycleway|footway|path|pedestrian|steps' + "|track|proposed|construction|bridleway|abandoned" + '|platform|raceway|service"]' + '["motor_vehicle"!~"no"]["motorcar"!~"no"]' + '["service"!~"parking|parking_aisle|driveway' + '|emergency_access"]' + ) # walk: select only roads and pathways that allow pedestrian access both # private and public pathways and roads. # Filter out limited access roadways and allow service roads - filters['walk'] = ('["highway"!~"motor|proposed|construction|abandoned' - '|platform|raceway"]["foot"!~"no"]' - '["pedestrians"!~"no"]') + filters["walk"] = ( + '["highway"!~"motor|proposed|construction|abandoned' + '|platform|raceway"]["foot"!~"no"]' + '["pedestrians"!~"no"]' + ) if network_type in filters: osm_filter = filters[network_type] @@ -67,10 +71,17 @@ def osm_filter(network_type): return osm_filter -def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, - network_type='walk', timeout=180, memory=None, - max_query_area_size=50*1000*50*1000, - custom_osm_filter=None): +def osm_net_download( + lat_min=None, + lng_min=None, + lat_max=None, + lng_max=None, + network_type="walk", + timeout=180, + memory=None, + max_query_area_size=50 * 1000 * 50 * 1000, + custom_osm_filter=None, +): """ Download OSM ways and nodes within a bounding box from the Overpass API. @@ -122,9 +133,9 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, # server memory allocation in bytes formatted for Overpass API query if memory is None: - maxsize = '' + maxsize = "" else: - maxsize = '[maxsize:{}]'.format(memory) + maxsize = "[maxsize:{}]".format(memory) # define the Overpass API query # way["highway"] denotes ways with highway keys and {filters} returns @@ -132,19 +143,23 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, # ways and way nodes. maxsize is in bytes. # turn bbox into a polygon and project to local UTM - polygon = Polygon([(lng_max, lat_min), (lng_min, lat_min), - (lng_min, lat_max), (lng_max, lat_max)]) - geometry_proj, crs_proj = project_geometry(polygon, - crs={'init': 'epsg:4326'}) + polygon = Polygon( + [(lng_max, lat_min), (lng_min, lat_min), (lng_min, lat_max), (lng_max, lat_max)] + ) + geometry_proj, crs_proj = project_geometry(polygon, crs={"init": "epsg:4326"}) # subdivide the bbox area poly if it exceeds the max area size # (in meters), then project back to WGS84 geometry_proj_consolidated_subdivided = consolidate_subdivide_geometry( - geometry_proj, max_query_area_size=max_query_area_size) - geometry, crs = project_geometry(geometry_proj_consolidated_subdivided, - crs=crs_proj, to_latlong=True) - log('Requesting network data within bounding box from Overpass API ' - 'in {:,} request(s)'.format(len(geometry))) + geometry_proj, max_query_area_size=max_query_area_size + ) + geometry, crs = project_geometry( + geometry_proj_consolidated_subdivided, crs=crs_proj, to_latlong=True + ) + log( + "Requesting network data within bounding box from Overpass API " + "in {:,} request(s)".format(len(geometry)) + ) start_time = time.time() # loop through each polygon in the geometry @@ -153,27 +168,35 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, # lat-longs to 8 decimal places to create # consistent URL strings lng_max, lat_min, lng_min, lat_max = poly.bounds - query_template = '[out:json][timeout:{timeout}]{maxsize};' \ - '(way["highway"]' \ - '{filters}({lat_min:.8f},{lng_max:.8f},' \ - '{lat_max:.8f},{lng_min:.8f});>;);out;' - query_str = query_template.format(lat_max=lat_max, lat_min=lat_min, - lng_min=lng_min, lng_max=lng_max, - filters=request_filter, - timeout=timeout, maxsize=maxsize) - response_json = overpass_request(data={'data': query_str}, - timeout=timeout) + query_template = ( + "[out:json][timeout:{timeout}]{maxsize};" + '(way["highway"]' + "{filters}({lat_min:.8f},{lng_max:.8f}," + "{lat_max:.8f},{lng_min:.8f});>;);out;" + ) + query_str = query_template.format( + lat_max=lat_max, + lat_min=lat_min, + lng_min=lng_min, + lng_max=lng_max, + filters=request_filter, + timeout=timeout, + maxsize=maxsize, + ) + response_json = overpass_request(data={"data": query_str}, timeout=timeout) response_jsons_list.append(response_json) - log('Downloaded OSM network data within bounding box from Overpass ' - 'API in {:,} request(s) and' - ' {:,.2f} seconds'.format(len(geometry), time.time()-start_time)) + log( + "Downloaded OSM network data within bounding box from Overpass " + "API in {:,} request(s) and" + " {:,.2f} seconds".format(len(geometry), time.time() - start_time) + ) # stitch together individual json results for json in response_jsons_list: try: - response_jsons.extend(json['elements']) + response_jsons.extend(json["elements"]) except KeyError: pass @@ -182,27 +205,30 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, record_count = len(response_jsons) if record_count == 0: - raise Exception('Query resulted in no data. Check your query ' - 'parameters: {}'.format(query_str)) + raise Exception( + "Query resulted in no data. Check your query " + "parameters: {}".format(query_str) + ) else: - response_jsons_df = pd.DataFrame.from_records(response_jsons, - index='id') - nodes = response_jsons_df[response_jsons_df['type'] == 'node'] - nodes = nodes[~nodes.index.duplicated(keep='first')] - ways = response_jsons_df[response_jsons_df['type'] == 'way'] - ways = ways[~ways.index.duplicated(keep='first')] + response_jsons_df = pd.DataFrame.from_records(response_jsons, index="id") + nodes = response_jsons_df[response_jsons_df["type"] == "node"] + nodes = nodes[~nodes.index.duplicated(keep="first")] + ways = response_jsons_df[response_jsons_df["type"] == "way"] + ways = ways[~ways.index.duplicated(keep="first")] response_jsons_df = pd.concat([nodes, ways], axis=0) response_jsons_df.reset_index(inplace=True) - response_jsons = response_jsons_df.to_dict(orient='records') + response_jsons = response_jsons_df.to_dict(orient="records") if record_count - len(response_jsons) > 0: - log('{:,} duplicate records removed. Took {:,.2f} seconds'.format( - record_count - len(response_jsons), time.time() - start_time)) + log( + "{:,} duplicate records removed. Took {:,.2f} seconds".format( + record_count - len(response_jsons), time.time() - start_time + ) + ) - return {'elements': response_jsons} + return {"elements": response_jsons} -def overpass_request(data, pause_duration=None, timeout=180, - error_pause_duration=None): +def overpass_request(data, pause_duration=None, timeout=180, error_pause_duration=None): """ Send a request to the Overpass API via HTTP POST and return the JSON response @@ -226,23 +252,25 @@ def overpass_request(data, pause_duration=None, timeout=180, """ # define the Overpass API URL, then construct a GET-style URL - url = 'http://www.overpass-api.de/api/interpreter' + url = "http://www.overpass-api.de/api/interpreter" start_time = time.time() log('Posting to {} with timeout={}, "{}"'.format(url, timeout, data)) response = requests.post(url, data=data, timeout=timeout) # get the response size and the domain, log result - size_kb = len(response.content) / 1000. - domain = re.findall(r'//(?s)(.*?)/', url)[0] - log('Downloaded {:,.1f}KB from {} in {:,.2f} seconds' - .format(size_kb, domain, time.time()-start_time)) + size_kb = len(response.content) / 1000.0 + domain = re.findall(r"//(?s)(.*?)/", url)[0] + log( + "Downloaded {:,.1f}KB from {} in {:,.2f} seconds".format( + size_kb, domain, time.time() - start_time + ) + ) try: response_json = response.json() - if 'remark' in response_json: - log('Server remark: "{}"'.format(response_json['remark'], - level=lg.WARNING)) + if "remark" in response_json: + log('Server remark: "{}"'.format(response_json["remark"], level=lg.WARNING)) except Exception: # 429 = 'too many requests' and 504 = 'gateway timeout' from server @@ -252,21 +280,31 @@ def overpass_request(data, pause_duration=None, timeout=180, # pause for error_pause_duration seconds before re-trying request if error_pause_duration is None: error_pause_duration = get_pause_duration() - log('Server at {} returned status code {} and no JSON data. ' - 'Re-trying request in {:.2f} seconds.' - .format(domain, response.status_code, error_pause_duration), - level=lg.WARNING) + log( + "Server at {} returned status code {} and no JSON data. " + "Re-trying request in {:.2f} seconds.".format( + domain, response.status_code, error_pause_duration + ), + level=lg.WARNING, + ) time.sleep(error_pause_duration) - response_json = overpass_request(data=data, - pause_duration=pause_duration, - timeout=timeout) + response_json = overpass_request( + data=data, pause_duration=pause_duration, timeout=timeout + ) # else, this was an unhandled status_code, throw an exception else: - log('Server at {} returned status code {} and no JSON data' - .format(domain, response.status_code), level=lg.ERROR) - raise Exception('Server returned no JSON data.\n{} {}\n{}' - .format(response, response.reason, response.text)) + log( + "Server at {} returned status code {} and no JSON data".format( + domain, response.status_code + ), + level=lg.ERROR, + ) + raise Exception( + "Server returned no JSON data.\n{} {}\n{}".format( + response, response.reason, response.text + ) + ) return response_json @@ -289,14 +327,13 @@ def get_pause_duration(recursive_delay=5, default_duration=10): pause_duration : int """ try: - response = requests.get('http://overpass-api.de/api/status') - status = response.text.split('\n')[3] - status_first_token = status.split(' ')[0] + response = requests.get("http://overpass-api.de/api/status") + status = response.text.split("\n")[3] + status_first_token = status.split(" ")[0] except Exception: # if status endpoint cannot be reached or output parsed, log error # and return default duration - log('Unable to query http://overpass-api.de/api/status', - level=lg.ERROR) + log("Unable to query http://overpass-api.de/api/status", level=lg.ERROR) return default_duration try: @@ -306,24 +343,24 @@ def get_pause_duration(recursive_delay=5, default_duration=10): pause_duration = 0 except Exception: # if first token is 'Slot', it tells you when your slot will be free - if status_first_token == 'Slot': - utc_time_str = status.split(' ')[3] + if status_first_token == "Slot": + utc_time_str = status.split(" ")[3] utc_time = date_parser.parse(utc_time_str).replace(tzinfo=None) pause_duration = math.ceil( - (utc_time - dt.datetime.utcnow()).total_seconds()) + (utc_time - dt.datetime.utcnow()).total_seconds() + ) pause_duration = max(pause_duration, 1) # if first token is 'Currently', it is currently running a query so # check back in recursive_delay seconds - elif status_first_token == 'Currently': + elif status_first_token == "Currently": time.sleep(recursive_delay) pause_duration = get_pause_duration() else: # any other status is unrecognized - log an error and return # default duration - log('Unrecognized server status: "{}"'.format(status), - level=lg.ERROR) + log('Unrecognized server status: "{}"'.format(status), level=lg.ERROR) return default_duration return pause_duration @@ -354,14 +391,13 @@ def consolidate_subdivide_geometry(geometry, max_query_area_size): quadrat_width = math.sqrt(max_query_area_size) if not isinstance(geometry, (Polygon, MultiPolygon)): - raise ValueError('Geometry must be a Shapely Polygon or MultiPolygon') + raise ValueError("Geometry must be a Shapely Polygon or MultiPolygon") # if geometry is a MultiPolygon OR a single Polygon whose area exceeds # the max size, get the convex hull around the geometry - if isinstance( - geometry, MultiPolygon) or \ - (isinstance( - geometry, Polygon) and geometry.area > max_query_area_size): + if isinstance(geometry, MultiPolygon) or ( + isinstance(geometry, Polygon) and geometry.area > max_query_area_size + ): geometry = geometry.convex_hull # if geometry area exceeds max size, subdivide it into smaller sub-polygons @@ -374,8 +410,7 @@ def consolidate_subdivide_geometry(geometry, max_query_area_size): return geometry -def quadrat_cut_geometry(geometry, quadrat_width, min_num=3, - buffer_amount=1e-9): +def quadrat_cut_geometry(geometry, quadrat_width, min_num=3, buffer_amount=1e-9): """ Split a Polygon or MultiPolygon up into sub-polygons of a specified size, using quadrats. @@ -400,16 +435,18 @@ def quadrat_cut_geometry(geometry, quadrat_width, min_num=3, # create n evenly spaced points between the min and max x and y bounds lng_max, lat_min, lng_min, lat_max = geometry.bounds - x_num = math.ceil((lng_min-lng_max) / quadrat_width) + 1 - y_num = math.ceil((lat_max-lat_min) / quadrat_width) + 1 + x_num = math.ceil((lng_min - lng_max) / quadrat_width) + 1 + y_num = math.ceil((lat_max - lat_min) / quadrat_width) + 1 x_points = np.linspace(lng_max, lng_min, num=max(x_num, min_num)) y_points = np.linspace(lat_min, lat_max, num=max(y_num, min_num)) # create a quadrat grid of lines at each of the evenly spaced points - vertical_lines = [LineString([(x, y_points[0]), (x, y_points[-1])]) - for x in x_points] - horizont_lines = [LineString([(x_points[0], y), (x_points[-1], y)]) - for y in y_points] + vertical_lines = [ + LineString([(x, y_points[0]), (x, y_points[-1])]) for x in x_points + ] + horizont_lines = [ + LineString([(x_points[0], y), (x_points[-1], y)]) for y in y_points + ] lines = vertical_lines + horizont_lines # buffer each line to distance of the quadrat width divided by 1 billion, @@ -443,75 +480,69 @@ def project_geometry(geometry, crs, to_latlong=False): """ gdf = gpd.GeoDataFrame() gdf.crs = crs - gdf.name = 'geometry to project' - gdf['geometry'] = None - gdf.loc[0, 'geometry'] = geometry + gdf.name = "geometry to project" + gdf["geometry"] = None + gdf.loc[0, "geometry"] = geometry gdf_proj = project_gdf(gdf, to_latlong=to_latlong) - geometry_proj = gdf_proj['geometry'].iloc[0] + geometry_proj = gdf_proj["geometry"].iloc[0] return geometry_proj, gdf_proj.crs -def project_gdf(gdf, to_latlong=False, verbose=False): +def project_gdf(gdf, to_crs=None, to_latlong=False): """ - Project a GeoDataFrame to the UTM zone appropriate for its geometries' - centroid. The calculation works well for most latitudes, - however it will not work well for some far northern locations. + Project a GeoDataFrame from its current CRS to another. + If to_crs is None, project to the UTM CRS for the UTM zone in which the + GeoDataFrame's centroid lies. Otherwise project to the CRS defined by + to_crs. The simple UTM zone calculation in this function works well for + most latitudes, but may not work for some extreme northern locations like + Svalbard or far northern Norway. If the GeoDataFrame is already in UTM, it + will be returned untouched. Parameters ---------- - gdf : GeoDataFrame - the gdf to be projected to UTM - to_latlong : bool, optional - if True, projects to WGS84 instead of to UTM - verbose : bool, optional - if False, turns off log and print statements for this function + gdf : geopandas.GeoDataFrame + the GeoDataFrame to be projected + to_crs : dict or string or pyproj.CRS + if None, project to UTM zone in which gdf's centroid lies, otherwise + project to this CRS + to_latlong : bool + if True, project to epsg:4326 and ignore to_crs Returns ------- - projected_gdf : GeoDataFrame + gdf_proj : geopandas.GeoDataFrame + the projected GeoDataFrame """ - assert len(gdf) > 0, 'You cannot project an empty GeoDataFrame.' - start_time = time.time() + if gdf.crs is None or len(gdf) < 1: + raise ValueError("GeoDataFrame must have a valid CRS and cannot be empty") + + # if to_latlong is True, project the gdf to latlong if to_latlong: - # if to_latlong is True, project the gdf to WGS84 - latlong_crs = {'init': 'epsg:4326'} - projected_gdf = gdf.to_crs(latlong_crs) - if not hasattr(gdf, 'name'): - gdf.name = 'unnamed' - if verbose: - log('Projected the GeoDataFrame "{}" to EPSG 4326 in {:,.2f} ' - 'seconds'.format(gdf.name, time.time()-start_time)) + gdf_proj = gdf.to_crs(4326) + + # else if to_crs was passed-in, project gdf to this CRS + elif to_crs is not None: + gdf_proj = gdf.to_crs(to_crs) + + # otherwise, automatically project the gdf to UTM else: - # else, project the gdf to UTM - # if GeoDataFrame is already in UTM, return it - if (gdf.crs is not None) and ('proj' in gdf.crs) \ - and (gdf.crs['proj'] == 'utm'): - return gdf - - # calculate the centroid of the union of all the geometries in the - # GeoDataFrame - avg_longitude = gdf['geometry'].unary_union.centroid.x - - # calculate the UTM zone from this avg longitude and define the - # UTM CRS to project - utm_zone = int(math.floor((avg_longitude + 180) / 6.) + 1) - utm_crs = {'datum': 'NAD83', - 'ellps': 'GRS80', - 'proj': 'utm', - 'zone': utm_zone, - 'units': 'm'} + if gdf.crs.is_projected: + raise ValueError("Geometry must be unprojected to calculate UTM zone") + + # calculate longitude of centroid of union of all geometries in gdf + avg_lng = gdf["geometry"].unary_union.centroid.x + + # calculate UTM zone from avg longitude to define CRS to project to + utm_zone = int(math.floor((avg_lng + 180) / 6.0) + 1) + utm_crs = ( + f"+proj=utm +zone={utm_zone} +ellps=WGS84 +datum=WGS84 +units=m +no_defs" + ) # project the GeoDataFrame to the UTM CRS - projected_gdf = gdf.to_crs(utm_crs) - if not hasattr(gdf, 'name'): - gdf.name = 'unnamed' - if verbose: - log('Projected the GeoDataFrame "{}" to UTM-{} in {:,.2f} ' - 'seconds'.format(gdf.name, utm_zone, time.time()-start_time)) + gdf_proj = gdf.to_crs(utm_crs) - projected_gdf.name = gdf.name - return projected_gdf + return gdf_proj def process_node(e): @@ -529,13 +560,11 @@ def process_node(e): node : dict """ - node = {'id': e['id'], - 'lat': e['lat'], - 'lon': e['lon']} + node = {"id": e["id"], "lat": e["lat"], "lon": e["lon"]} - if 'tags' in e: - if e['tags'] is not np.nan: - for t, v in list(e['tags'].items()): + if "tags" in e: + if e["tags"] is not np.nan: + for t, v in list(e["tags"].items()): if t in config.settings.keep_osm_tags: node[t] = v @@ -558,19 +587,19 @@ def process_way(e): waynodes : list of dict """ - way = {'id': e['id']} + way = {"id": e["id"]} - if 'tags' in e: - if e['tags'] is not np.nan: - for t, v in list(e['tags'].items()): + if "tags" in e: + if e["tags"] is not np.nan: + for t, v in list(e["tags"].items()): if t in config.settings.keep_osm_tags: way[t] = v # nodes that make up a way waynodes = [] - for n in e['nodes']: - waynodes.append({'way_id': e['id'], 'node_id': n}) + for n in e["nodes"]: + waynodes.append({"way_id": e["id"], "node_id": n}) return way, waynodes @@ -590,32 +619,39 @@ def parse_network_osm_query(data): nodes, ways, waynodes as a tuple of pandas.DataFrames """ - if len(data['elements']) == 0: - raise RuntimeError('OSM query results contain no data.') + if len(data["elements"]) == 0: + raise RuntimeError("OSM query results contain no data.") nodes = [] ways = [] waynodes = [] - for e in data['elements']: - if e['type'] == 'node': + for e in data["elements"]: + if e["type"] == "node": nodes.append(process_node(e)) - elif e['type'] == 'way': + elif e["type"] == "way": w, wn = process_way(e) ways.append(w) waynodes.extend(wn) - nodes = pd.DataFrame.from_records(nodes, index='id') - ways = pd.DataFrame.from_records(ways, index='id') - waynodes = pd.DataFrame.from_records(waynodes, index='way_id') + nodes = pd.DataFrame.from_records(nodes, index="id") + ways = pd.DataFrame.from_records(ways, index="id") + waynodes = pd.DataFrame.from_records(waynodes, index="way_id") return (nodes, ways, waynodes) -def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, - timeout=180, memory=None, - max_query_area_size=50*1000*50*1000, - custom_osm_filter=None): +def ways_in_bbox( + lat_min, + lng_min, + lat_max, + lng_max, + network_type, + timeout=180, + memory=None, + max_query_area_size=50 * 1000 * 50 * 1000, + custom_osm_filter=None, +): """ Get DataFrames of OSM data in a bounding box. @@ -655,11 +691,18 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, """ return parse_network_osm_query( - osm_net_download(lat_max=lat_max, lat_min=lat_min, lng_min=lng_min, - lng_max=lng_max, network_type=network_type, - timeout=timeout, memory=memory, - max_query_area_size=max_query_area_size, - custom_osm_filter=custom_osm_filter)) + osm_net_download( + lat_max=lat_max, + lat_min=lat_min, + lng_min=lng_min, + lng_max=lng_max, + network_type=network_type, + timeout=timeout, + memory=memory, + max_query_area_size=max_query_area_size, + custom_osm_filter=custom_osm_filter, + ) + ) def intersection_nodes(waynodes): @@ -710,6 +753,7 @@ def node_pairs(nodes, ways, waynodes, two_way=True): def pairwise(l): return zip(islice(l, 0, len(l)), islice(l, 1, None)) + intersections = intersection_nodes(waynodes) waymap = waynodes.groupby(level=0, sort=False) pairs = [] @@ -729,9 +773,11 @@ def pairwise(l): distance = round(gcd(fn.lat, fn.lon, tn.lat, tn.lon), 6) - col_dict = {'from_id': from_node, - 'to_id': to_node, - 'distance': distance} + col_dict = { + "from_id": from_node, + "to_id": to_node, + "distance": distance, + } for tag in config.settings.keep_osm_tags: try: @@ -743,9 +789,11 @@ def pairwise(l): if not two_way: - col_dict = {'from_id': to_node, - 'to_id': from_node, - 'distance': distance} + col_dict = { + "from_id": to_node, + "to_id": from_node, + "distance": distance, + } for tag in config.settings.keep_osm_tags: try: @@ -757,22 +805,36 @@ def pairwise(l): pairs = pd.DataFrame.from_records(pairs) if pairs.empty: - raise Exception('Query resulted in no connected node pairs. Check ' - 'your query parameters or bounding box') + raise Exception( + "Query resulted in no connected node pairs. Check " + "your query parameters or bounding box" + ) else: - pairs.index = pd.MultiIndex.from_arrays([pairs['from_id'].values, - pairs['to_id'].values]) - log('Edge node pairs completed. Took {:,.2f} seconds' - .format(time.time()-start_time)) + pairs.index = pd.MultiIndex.from_arrays( + [pairs["from_id"].values, pairs["to_id"].values] + ) + log( + "Edge node pairs completed. Took {:,.2f} seconds".format( + time.time() - start_time + ) + ) return pairs -def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, - bbox=None, network_type='walk', two_way=True, - timeout=180, memory=None, - max_query_area_size=50*1000*50*1000, - custom_osm_filter=None): +def network_from_bbox( + lat_min=None, + lng_min=None, + lat_max=None, + lng_max=None, + bbox=None, + network_type="walk", + two_way=True, + timeout=180, + memory=None, + max_query_area_size=50 * 1000 * 50 * 1000, + custom_osm_filter=None, +): """ Make a graph network from a bounding lat/lon box composed of nodes and edges for use in Pandana street network accessibility calculations. @@ -834,44 +896,68 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, start_time = time.time() if bbox is not None: - assert isinstance(bbox, tuple) \ - and len(bbox) == 4, 'bbox must be a 4 element tuple' - assert (lat_min is None) and (lng_min is None) and \ - (lat_max is None) and (lng_max is None), \ - 'lat_min, lng_min, lat_max and lng_max must be None ' \ - 'if you are using bbox' + assert ( + isinstance(bbox, tuple) and len(bbox) == 4 + ), "bbox must be a 4 element tuple" + assert ( + (lat_min is None) + and (lng_min is None) + and (lat_max is None) + and (lng_max is None) + ), ( + "lat_min, lng_min, lat_max and lng_max must be None " + "if you are using bbox" + ) lng_max, lat_min, lng_min, lat_max = bbox - assert lat_min is not None, 'lat_min cannot be None' - assert lng_min is not None, 'lng_min cannot be None' - assert lat_max is not None, 'lat_max cannot be None' - assert lng_max is not None, 'lng_max cannot be None' - assert isinstance(lat_min, float) and isinstance(lng_min, float) and \ - isinstance(lat_max, float) and isinstance(lng_max, float), \ - 'lat_min, lng_min, lat_max, and lng_max must be floats' + assert lat_min is not None, "lat_min cannot be None" + assert lng_min is not None, "lng_min cannot be None" + assert lat_max is not None, "lat_max cannot be None" + assert lng_max is not None, "lng_max cannot be None" + assert ( + isinstance(lat_min, float) + and isinstance(lng_min, float) + and isinstance(lat_max, float) + and isinstance(lng_max, float) + ), "lat_min, lng_min, lat_max, and lng_max must be floats" nodes, ways, waynodes = ways_in_bbox( - lat_min=lat_min, lng_min=lng_min, lat_max=lat_max, lng_max=lng_max, - network_type=network_type, timeout=timeout, - memory=memory, max_query_area_size=max_query_area_size, - custom_osm_filter=custom_osm_filter) - log('Returning OSM data with {:,} nodes and {:,} ways...' - .format(len(nodes), len(ways))) + lat_min=lat_min, + lng_min=lng_min, + lat_max=lat_max, + lng_max=lng_max, + network_type=network_type, + timeout=timeout, + memory=memory, + max_query_area_size=max_query_area_size, + custom_osm_filter=custom_osm_filter, + ) + log( + "Returning OSM data with {:,} nodes and {:,} ways...".format( + len(nodes), len(ways) + ) + ) edgesfinal = node_pairs(nodes, ways, waynodes, two_way=two_way) # make the unique set of nodes that ended up in pairs - node_ids = sorted(set(edgesfinal['from_id'].unique()) - .union(set(edgesfinal['to_id'].unique()))) + node_ids = sorted( + set(edgesfinal["from_id"].unique()).union(set(edgesfinal["to_id"].unique())) + ) nodesfinal = nodes.loc[node_ids] - nodesfinal = nodesfinal[['lon', 'lat']] - nodesfinal.rename(columns={'lon': 'x', 'lat': 'y'}, inplace=True) - nodesfinal['id'] = nodesfinal.index - edgesfinal.rename(columns={'from_id': 'from', 'to_id': 'to'}, inplace=True) - log('Returning processed graph with {:,} nodes and {:,} edges...' - .format(len(nodesfinal), len(edgesfinal))) - log('Completed OSM data download and Pandana node and edge table ' - 'creation in {:,.2f} seconds'.format(time.time()-start_time)) + nodesfinal = nodesfinal[["lon", "lat"]] + nodesfinal.rename(columns={"lon": "x", "lat": "y"}, inplace=True) + nodesfinal["id"] = nodesfinal.index + edgesfinal.rename(columns={"from_id": "from", "to_id": "to"}, inplace=True) + log( + "Returning processed graph with {:,} nodes and {:,} edges...".format( + len(nodesfinal), len(edgesfinal) + ) + ) + log( + "Completed OSM data download and Pandana node and edge table " + "creation in {:,.2f} seconds".format(time.time() - start_time) + ) return nodesfinal, edgesfinal From 38bdbb0e1948521af71f75246766f2b45dc57a11 Mon Sep 17 00:00:00 2001 From: eli knaap Date: Tue, 30 Jun 2020 09:19:21 -0700 Subject: [PATCH 02/25] bump min geopandas --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3473523..d9de93b 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ 'requests >= 2.9.1', 'pandas >= 0.16.0', 'numpy>=1.10', - 'geopandas>=0.2.1', + 'geopandas>=0.7', 'Shapely>=1.5' ] ) From 0b39668f19f59b250a3a8c22ab17b4b5a86c3cf2 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 14:08:54 -0700 Subject: [PATCH 03/25] Updating travis script --- .coveragerc | 4 ---- .travis.yml | 11 +++-------- requirements-dev.txt | 7 +++++-- 3 files changed, 8 insertions(+), 14 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 8d32801..0000000 --- a/.coveragerc +++ /dev/null @@ -1,4 +0,0 @@ -[run] -omit = - osmnet/**/tests/* - */__init__.py diff --git a/.travis.yml b/.travis.yml index c68e101..ee4969c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,8 @@ python: - '2.7' - '3.5' - '3.6' - -matrix: - include: - - python: "3.7" # temp solution to test in python 3.7 - dist: xenial - sudo: true + - '3.7' + - '3.8' install: - pip install . @@ -18,9 +14,8 @@ install: - pip show osmnet script: - - pycodestyle osmnet + - pycodestyle --max-line-length=100 osmnet - py.test --cov osmnet --cov-report term-missing after_success: - coveralls - - bin/build_docs.sh diff --git a/requirements-dev.txt b/requirements-dev.txt index d6dab9a..7d35ed4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,12 @@ -# requirements for development and testing +# Additional requirements for development and testing +# testing coveralls -numpydoc pycodestyle pytest pytest-cov + +# building documentation +numpydoc sphinx sphinx_rtd_theme From 92c02a04cdbd855a2cbc8ad97796eb82ee3d9f02 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 14:17:46 -0700 Subject: [PATCH 04/25] Pinning pytest-cov --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7d35ed4..168dbed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ coveralls pycodestyle pytest -pytest-cov +pytest-cov < 2.10 # building documentation numpydoc From 902042cab54111dc1b1f0c35e5ec28e96f2f4b2e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 14:19:16 -0700 Subject: [PATCH 05/25] Pycodestyle fixes --- osmnet/load.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index 66a0335..aaf3fe9 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -708,8 +708,8 @@ def node_pairs(nodes, ways, waynodes, two_way=True): """ start_time = time.time() - def pairwise(l): - return zip(islice(l, 0, len(l)), islice(l, 1, None)) + def pairwise(ls): + return zip(islice(ls, 0, len(ls)), islice(ls, 1, None)) intersections = intersection_nodes(waynodes) waymap = waynodes.groupby(level=0, sort=False) pairs = [] @@ -838,8 +838,8 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, and len(bbox) == 4, 'bbox must be a 4 element tuple' assert (lat_min is None) and (lng_min is None) and \ (lat_max is None) and (lng_max is None), \ - 'lat_min, lng_min, lat_max and lng_max must be None ' \ - 'if you are using bbox' + 'lat_min, lng_min, lat_max and lng_max must be None ' \ + 'if you are using bbox' lng_max, lat_min, lng_min, lat_max = bbox From 122b96dac443b4754084e99d3e00c20600ca4933 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 14:45:40 -0700 Subject: [PATCH 06/25] Updating versions and dates --- LICENSE.txt | 2 +- docs/source/conf.py | 6 +++--- osmnet/__init__.py | 2 +- setup.py | 15 +++++++++------ 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 5b16623..cf92d5c 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,7 +1,7 @@ GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 - Copyright (C) 2017 UrbanSim Inc. + Copyright (C) 2020 UrbanSim Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. diff --git a/docs/source/conf.py b/docs/source/conf.py index f0619e5..067367a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,10 +28,10 @@ source_suffix = '.rst' master_doc = 'index' project = u'OSMnet' -copyright = u'2019, UrbanSim Inc.' +copyright = u'2020, UrbanSim Inc.' author = u'UrbanSim Inc.' -version = u'0.1.5' -release = u'0.1.5' +version = u'0.1.6' +release = u'0.1.6' language = None nitpicky = True diff --git a/osmnet/__init__.py b/osmnet/__init__.py index 2e6ae7b..7b32f76 100644 --- a/osmnet/__init__.py +++ b/osmnet/__init__.py @@ -1,5 +1,5 @@ from .load import * -__version__ = "0.1.5" +__version__ = "0.1.6" version = __version__ diff --git a/setup.py b/setup.py index 3473523..2802e2f 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( name='osmnet', - version='0.1.5', + version='0.1.6', license='AGPL', description=('Tools for the extraction of OpenStreetMap street network ' 'data for use in Pandana accessibility analyses.'), @@ -22,19 +22,22 @@ author='UrbanSim Inc.', url='https://github.com/UDST/osmnet', classifiers=[ - 'Programming Language :: Python :: 2.7', + 'Intended Audience :: Science/Research', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Development Status :: 3 - Alpha', 'License :: OSI Approved :: GNU Affero General Public License v3' ], packages=find_packages(exclude=['*.tests']), install_requires=[ - 'requests >= 2.9.1', + 'geopandas >= 0.2.1', + 'numpy >= 1.10', 'pandas >= 0.16.0', - 'numpy>=1.10', - 'geopandas>=0.2.1', - 'Shapely>=1.5' + 'requests >= 2.9.1', + 'shapely >= 1.5' ] ) From 8aaccc00233645eb74110ccde35be361db0fbeba Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 14:47:53 -0700 Subject: [PATCH 07/25] Removing ez_setup --- setup.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/setup.py b/setup.py index 2802e2f..b95b214 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,5 @@ -# Install setuptools if not installed. -try: - import setuptools -except ImportError: - from ez_setup import use_setuptools - use_setuptools() - from setuptools import setup, find_packages - # read README as the long description with open('README.rst', 'r') as f: long_description = f.read() From 94a1a9da4463f6aa9ace5efdd88e63beb0f9465e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 14:50:42 -0700 Subject: [PATCH 08/25] Updating contribution info --- CONTRIBUTING.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a58914..a6d2d20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,12 +1,56 @@ -#### If you have found an error: +## If you have found an error: - check the error message and [documentation](https://udst.github.io/osmnet/index.html) - search the previously opened and closed issues to see if the problem has already been reported - if the problem is with a dependency of OSMnet, please open an issue on the dependency's repo - if the problem is with OSMnet and you think you may have a fix, please submit a PR, otherwise please open an issue in the [issue tracker](https://github.com/UDST/osmnet/issues) following the issue template -#### Making a feature proposal or contributing code: +## Making a feature proposal or contributing code: - post your requested feature on the [issue tracker](https://github.com/UDST/osmnet/issues) and mark it with a `New feature` label so it can be reviewed - fork the repo, make your change (your code should attempt to conform to OSMnet's existing coding, commenting, and docstring styles), add new or update [unit tests](https://github.com/UDST/osmnet/tree/master/osmnet/tests), and submit a PR - respond to the code review +## Updating the documentation: + +- See instructions in `docs/README.md` + + +## Preparing a release: + +- Make a new branch for release prep + +- Update the version number and changelog + - `CHANGELOG.md` + - `setup.py` + - `osmnet/__init__.py` + - `docs/source/index.rst` + +- Make sure all the tests are passing, and check if updates are needed to `README.md` or to the documentation + +- Open a pull request to the master branch to finalize it + +- After merging, tag the release on GitHub and follow the distribution procedures below + + +## Distributing a release on PyPI (for pip installation): + +- Register an account at https://pypi.org, ask one of the current maintainers to add you to the project, and `pip install twine` + +- Check out the copy of the code you'd like to release + +- Run `python setup.py sdist bdist_wheel --universal` + +- This should create a `dist` directory containing two package files -- delete any old ones before the next step + +- Run `twine upload dist/*` -- this will prompt you for your pypi.org credentials + +- Check https://pypi.org/project/osmnet/ for the new version + + +## Distributing a release on Conda Forge (for conda installation): + +- The [conda-forge/osmnet-feedstock](https://github.com/conda-forge/osmnet-feedstock) repository controls the Conda Forge release + +- Conda Forge bots usually detect new releases on PyPI and set in motion the appropriate feedstock updates, which a current maintainer will need to approve and merge + +- Check https://anaconda.org/conda-forge/osmnet for the new version (may take a few minutes for it to appear) \ No newline at end of file From db99da806e6b7468fb80bdac6a500a21e6d31d14 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 14:54:40 -0700 Subject: [PATCH 09/25] Dropping python 2.7 --- .travis.yml | 1 - setup.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ee4969c..e66eb8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - - '2.7' - '3.5' - '3.6' - '3.7' diff --git a/setup.py b/setup.py index b95b214..cdb5b41 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ 'License :: OSI Approved :: GNU Affero General Public License v3' ], packages=find_packages(exclude=['*.tests']), + python_requires='>=3', install_requires=[ 'geopandas >= 0.2.1', 'numpy >= 1.10', From 2675cf0f0f7b9abddb76ceb7aec017014060a696 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 14:59:49 -0700 Subject: [PATCH 10/25] Updating doc instructions --- docs/Makefile | 20 -------------------- docs/README.md | 31 +++++++++++++++++++++++++++++++ docs/build/.gitignore | 1 + docs/make.bat | 36 ------------------------------------ 4 files changed, 32 insertions(+), 56 deletions(-) delete mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/build/.gitignore delete mode 100644 docs/make.bat diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 6f44cb0..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = OSMnet -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..ab8ccf0 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,31 @@ +This folder generates the OSMNet online documentation, hosted at https://udst.github.io/osmnet/. + +### How it works + +HTML files are generated using [Sphinx](http://sphinx-doc.org) and hosted with GitHub Pages from the `gh-pages` branch of the repository. The online documentation is rendered and updated **manually**. + +### Editing the documentation + +The files in `docs/source`, along with docstrings in the source code, determine what appears in the rendered documentation. Here's a [good tutorial](https://pythonhosted.org/an_example_pypi_project/sphinx.html) for Sphinx. + +### Previewing changes locally + +Install the copy of OSMNet that the documentation is meant to reflect. Install the documentation tools. + +``` +pip install . +pip install sphinx sphinx_rtd_theme numpydoc +``` + +Build the documentation. There should be status messages and warnings, but no errors. + +``` +cd docs +sphinx-build -b html source build +``` + +The HTML files will show up in `docs/build/`. + +### Uploading changes + +Clone a second copy of the repository and check out the `gh-pages` branch. Copy over the updated HTML files, commit them, and push the changes to GitHub. diff --git a/docs/build/.gitignore b/docs/build/.gitignore new file mode 100644 index 0000000..df3359d --- /dev/null +++ b/docs/build/.gitignore @@ -0,0 +1 @@ +*/* \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index bbf11d0..0000000 --- a/docs/make.bat +++ /dev/null @@ -1,36 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build -set SPHINXPROJ=OSMnet - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd From e7d707f94c6cbf923aecf705eec63e7834844ba4 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 8 Jul 2020 15:27:47 -0700 Subject: [PATCH 11/25] Typo in readme --- README.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.rst b/README.rst index a7b630b..c2df2e1 100644 --- a/README.rst +++ b/README.rst @@ -59,11 +59,7 @@ OSMnet can be installed via PyPI: Development Installation ^^^^^^^^^^^^^^^^^^^^^^^^ -To install use the ``develop`` command rather than ``install``. Make sure you -are using the latest version of the code base by using git’s ``git pull`` -inside the cloned repository. - -To install OSMnet follow these steps: +To install OSMnet from source code, follow these steps: 1. Git clone the `OSMnet repo`_ 2. in the cloned directory run: ``python setup.py develop`` From f74b183a5813da43c3665c42ab08a8242aea4d10 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 9 Jul 2020 10:31:34 -0700 Subject: [PATCH 12/25] Additional merge conflict resolutions --- osmnet/load.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index 5d58d56..7c8a94a 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -751,14 +751,9 @@ def node_pairs(nodes, ways, waynodes, two_way=True): """ start_time = time.time() -<<<<<<< HEAD def pairwise(ls): return zip(islice(ls, 0, len(ls)), islice(ls, 1, None)) -======= - def pairwise(l): - return zip(islice(l, 0, len(l)), islice(l, 1, None)) ->>>>>>> dev intersections = intersection_nodes(waynodes) waymap = waynodes.groupby(level=0, sort=False) pairs = [] @@ -901,27 +896,12 @@ def network_from_bbox( start_time = time.time() if bbox is not None: -<<<<<<< HEAD assert isinstance(bbox, tuple) \ and len(bbox) == 4, 'bbox must be a 4 element tuple' assert (lat_min is None) and (lng_min is None) and \ (lat_max is None) and (lng_max is None), \ 'lat_min, lng_min, lat_max and lng_max must be None ' \ 'if you are using bbox' -======= - assert ( - isinstance(bbox, tuple) and len(bbox) == 4 - ), "bbox must be a 4 element tuple" - assert ( - (lat_min is None) - and (lng_min is None) - and (lat_max is None) - and (lng_max is None) - ), ( - "lat_min, lng_min, lat_max and lng_max must be None " - "if you are using bbox" - ) ->>>>>>> dev lng_max, lat_min, lng_min, lat_max = bbox From d9711e57f7c884df77b8ecf13e869a0252bce833 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 9 Jul 2020 10:50:09 -0700 Subject: [PATCH 13/25] Adding version info to docs --- docs/source/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 1e6c9e2..8c31009 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,6 +3,8 @@ OSMnet Tools for the extraction of `OpenStreetMap`_ (OSM) street network data. Intended to be used in tandem with `Pandana`_ and `UrbanAccess`_ libraries to extract street network nodes and edges. +v0.1.6, released July 9, 2020. + Contents -------- From 2c3dd8600750cbce1a1298afcce94ac261f0ef3a Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 9 Jul 2020 11:05:29 -0700 Subject: [PATCH 14/25] Reverting broken code formatting --- osmnet/load.py | 423 ++++++++++++++++++++----------------------------- 1 file changed, 168 insertions(+), 255 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index 7c8a94a..0534e33 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -45,23 +45,19 @@ def osm_filter(network_type): # passenger vehicles both private and public # roads. Filter out un-drivable roads and service roads tagged as parking, # driveway, or emergency-access - filters["drive"] = ( - '["highway"!~"cycleway|footway|path|pedestrian|steps' - "|track|proposed|construction|bridleway|abandoned" - '|platform|raceway|service"]' - '["motor_vehicle"!~"no"]["motorcar"!~"no"]' - '["service"!~"parking|parking_aisle|driveway' - '|emergency_access"]' - ) + filters['drive'] = ('["highway"!~"cycleway|footway|path|pedestrian|steps' + '|track|proposed|construction|bridleway|abandoned' + '|platform|raceway|service"]' + '["motor_vehicle"!~"no"]["motorcar"!~"no"]' + '["service"!~"parking|parking_aisle|driveway' + '|emergency_access"]') # walk: select only roads and pathways that allow pedestrian access both # private and public pathways and roads. # Filter out limited access roadways and allow service roads - filters["walk"] = ( - '["highway"!~"motor|proposed|construction|abandoned' - '|platform|raceway"]["foot"!~"no"]' - '["pedestrians"!~"no"]' - ) + filters['walk'] = ('["highway"!~"motor|proposed|construction|abandoned' + '|platform|raceway"]["foot"!~"no"]' + '["pedestrians"!~"no"]') if network_type in filters: osm_filter = filters[network_type] @@ -71,17 +67,10 @@ def osm_filter(network_type): return osm_filter -def osm_net_download( - lat_min=None, - lng_min=None, - lat_max=None, - lng_max=None, - network_type="walk", - timeout=180, - memory=None, - max_query_area_size=50 * 1000 * 50 * 1000, - custom_osm_filter=None, -): +def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, + network_type='walk', timeout=180, memory=None, + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Download OSM ways and nodes within a bounding box from the Overpass API. @@ -133,9 +122,9 @@ def osm_net_download( # server memory allocation in bytes formatted for Overpass API query if memory is None: - maxsize = "" + maxsize = '' else: - maxsize = "[maxsize:{}]".format(memory) + maxsize = '[maxsize:{}]'.format(memory) # define the Overpass API query # way["highway"] denotes ways with highway keys and {filters} returns @@ -143,23 +132,19 @@ def osm_net_download( # ways and way nodes. maxsize is in bytes. # turn bbox into a polygon and project to local UTM - polygon = Polygon( - [(lng_max, lat_min), (lng_min, lat_min), (lng_min, lat_max), (lng_max, lat_max)] - ) - geometry_proj, crs_proj = project_geometry(polygon, crs={"init": "epsg:4326"}) + polygon = Polygon([(lng_max, lat_min), (lng_min, lat_min), + (lng_min, lat_max), (lng_max, lat_max)]) + geometry_proj, crs_proj = project_geometry(polygon, + crs={'init': 'epsg:4326'}) # subdivide the bbox area poly if it exceeds the max area size # (in meters), then project back to WGS84 geometry_proj_consolidated_subdivided = consolidate_subdivide_geometry( - geometry_proj, max_query_area_size=max_query_area_size - ) - geometry, crs = project_geometry( - geometry_proj_consolidated_subdivided, crs=crs_proj, to_latlong=True - ) - log( - "Requesting network data within bounding box from Overpass API " - "in {:,} request(s)".format(len(geometry)) - ) + geometry_proj, max_query_area_size=max_query_area_size) + geometry, crs = project_geometry(geometry_proj_consolidated_subdivided, + crs=crs_proj, to_latlong=True) + log('Requesting network data within bounding box from Overpass API ' + 'in {:,} request(s)'.format(len(geometry))) start_time = time.time() # loop through each polygon in the geometry @@ -168,35 +153,27 @@ def osm_net_download( # lat-longs to 8 decimal places to create # consistent URL strings lng_max, lat_min, lng_min, lat_max = poly.bounds - query_template = ( - "[out:json][timeout:{timeout}]{maxsize};" - '(way["highway"]' - "{filters}({lat_min:.8f},{lng_max:.8f}," - "{lat_max:.8f},{lng_min:.8f});>;);out;" - ) - query_str = query_template.format( - lat_max=lat_max, - lat_min=lat_min, - lng_min=lng_min, - lng_max=lng_max, - filters=request_filter, - timeout=timeout, - maxsize=maxsize, - ) - response_json = overpass_request(data={"data": query_str}, timeout=timeout) + query_template = '[out:json][timeout:{timeout}]{maxsize};' \ + '(way["highway"]' \ + '{filters}({lat_min:.8f},{lng_max:.8f},' \ + '{lat_max:.8f},{lng_min:.8f});>;);out;' + query_str = query_template.format(lat_max=lat_max, lat_min=lat_min, + lng_min=lng_min, lng_max=lng_max, + filters=request_filter, + timeout=timeout, maxsize=maxsize) + response_json = overpass_request(data={'data': query_str}, + timeout=timeout) response_jsons_list.append(response_json) - log( - "Downloaded OSM network data within bounding box from Overpass " - "API in {:,} request(s) and" - " {:,.2f} seconds".format(len(geometry), time.time() - start_time) - ) + log('Downloaded OSM network data within bounding box from Overpass ' + 'API in {:,} request(s) and' + ' {:,.2f} seconds'.format(len(geometry), time.time()-start_time)) # stitch together individual json results for json in response_jsons_list: try: - response_jsons.extend(json["elements"]) + response_jsons.extend(json['elements']) except KeyError: pass @@ -205,30 +182,27 @@ def osm_net_download( record_count = len(response_jsons) if record_count == 0: - raise Exception( - "Query resulted in no data. Check your query " - "parameters: {}".format(query_str) - ) + raise Exception('Query resulted in no data. Check your query ' + 'parameters: {}'.format(query_str)) else: - response_jsons_df = pd.DataFrame.from_records(response_jsons, index="id") - nodes = response_jsons_df[response_jsons_df["type"] == "node"] - nodes = nodes[~nodes.index.duplicated(keep="first")] - ways = response_jsons_df[response_jsons_df["type"] == "way"] - ways = ways[~ways.index.duplicated(keep="first")] + response_jsons_df = pd.DataFrame.from_records(response_jsons, + index='id') + nodes = response_jsons_df[response_jsons_df['type'] == 'node'] + nodes = nodes[~nodes.index.duplicated(keep='first')] + ways = response_jsons_df[response_jsons_df['type'] == 'way'] + ways = ways[~ways.index.duplicated(keep='first')] response_jsons_df = pd.concat([nodes, ways], axis=0) response_jsons_df.reset_index(inplace=True) - response_jsons = response_jsons_df.to_dict(orient="records") + response_jsons = response_jsons_df.to_dict(orient='records') if record_count - len(response_jsons) > 0: - log( - "{:,} duplicate records removed. Took {:,.2f} seconds".format( - record_count - len(response_jsons), time.time() - start_time - ) - ) + log('{:,} duplicate records removed. Took {:,.2f} seconds'.format( + record_count - len(response_jsons), time.time() - start_time)) - return {"elements": response_jsons} + return {'elements': response_jsons} -def overpass_request(data, pause_duration=None, timeout=180, error_pause_duration=None): +def overpass_request(data, pause_duration=None, timeout=180, + error_pause_duration=None): """ Send a request to the Overpass API via HTTP POST and return the JSON response @@ -252,25 +226,23 @@ def overpass_request(data, pause_duration=None, timeout=180, error_pause_duratio """ # define the Overpass API URL, then construct a GET-style URL - url = "http://www.overpass-api.de/api/interpreter" + url = 'http://www.overpass-api.de/api/interpreter' start_time = time.time() log('Posting to {} with timeout={}, "{}"'.format(url, timeout, data)) response = requests.post(url, data=data, timeout=timeout) # get the response size and the domain, log result - size_kb = len(response.content) / 1000.0 - domain = re.findall(r"//(?s)(.*?)/", url)[0] - log( - "Downloaded {:,.1f}KB from {} in {:,.2f} seconds".format( - size_kb, domain, time.time() - start_time - ) - ) + size_kb = len(response.content) / 1000. + domain = re.findall(r'//(?s)(.*?)/', url)[0] + log('Downloaded {:,.1f}KB from {} in {:,.2f} seconds' + .format(size_kb, domain, time.time()-start_time)) try: response_json = response.json() - if "remark" in response_json: - log('Server remark: "{}"'.format(response_json["remark"], level=lg.WARNING)) + if 'remark' in response_json: + log('Server remark: "{}"'.format(response_json['remark'], + level=lg.WARNING)) except Exception: # 429 = 'too many requests' and 504 = 'gateway timeout' from server @@ -280,31 +252,21 @@ def overpass_request(data, pause_duration=None, timeout=180, error_pause_duratio # pause for error_pause_duration seconds before re-trying request if error_pause_duration is None: error_pause_duration = get_pause_duration() - log( - "Server at {} returned status code {} and no JSON data. " - "Re-trying request in {:.2f} seconds.".format( - domain, response.status_code, error_pause_duration - ), - level=lg.WARNING, - ) + log('Server at {} returned status code {} and no JSON data. ' + 'Re-trying request in {:.2f} seconds.' + .format(domain, response.status_code, error_pause_duration), + level=lg.WARNING) time.sleep(error_pause_duration) - response_json = overpass_request( - data=data, pause_duration=pause_duration, timeout=timeout - ) + response_json = overpass_request(data=data, + pause_duration=pause_duration, + timeout=timeout) # else, this was an unhandled status_code, throw an exception else: - log( - "Server at {} returned status code {} and no JSON data".format( - domain, response.status_code - ), - level=lg.ERROR, - ) - raise Exception( - "Server returned no JSON data.\n{} {}\n{}".format( - response, response.reason, response.text - ) - ) + log('Server at {} returned status code {} and no JSON data' + .format(domain, response.status_code), level=lg.ERROR) + raise Exception('Server returned no JSON data.\n{} {}\n{}' + .format(response, response.reason, response.text)) return response_json @@ -327,13 +289,14 @@ def get_pause_duration(recursive_delay=5, default_duration=10): pause_duration : int """ try: - response = requests.get("http://overpass-api.de/api/status") - status = response.text.split("\n")[3] - status_first_token = status.split(" ")[0] + response = requests.get('http://overpass-api.de/api/status') + status = response.text.split('\n')[3] + status_first_token = status.split(' ')[0] except Exception: # if status endpoint cannot be reached or output parsed, log error # and return default duration - log("Unable to query http://overpass-api.de/api/status", level=lg.ERROR) + log('Unable to query http://overpass-api.de/api/status', + level=lg.ERROR) return default_duration try: @@ -343,24 +306,24 @@ def get_pause_duration(recursive_delay=5, default_duration=10): pause_duration = 0 except Exception: # if first token is 'Slot', it tells you when your slot will be free - if status_first_token == "Slot": - utc_time_str = status.split(" ")[3] + if status_first_token == 'Slot': + utc_time_str = status.split(' ')[3] utc_time = date_parser.parse(utc_time_str).replace(tzinfo=None) pause_duration = math.ceil( - (utc_time - dt.datetime.utcnow()).total_seconds() - ) + (utc_time - dt.datetime.utcnow()).total_seconds()) pause_duration = max(pause_duration, 1) # if first token is 'Currently', it is currently running a query so # check back in recursive_delay seconds - elif status_first_token == "Currently": + elif status_first_token == 'Currently': time.sleep(recursive_delay) pause_duration = get_pause_duration() else: # any other status is unrecognized - log an error and return # default duration - log('Unrecognized server status: "{}"'.format(status), level=lg.ERROR) + log('Unrecognized server status: "{}"'.format(status), + level=lg.ERROR) return default_duration return pause_duration @@ -391,13 +354,14 @@ def consolidate_subdivide_geometry(geometry, max_query_area_size): quadrat_width = math.sqrt(max_query_area_size) if not isinstance(geometry, (Polygon, MultiPolygon)): - raise ValueError("Geometry must be a Shapely Polygon or MultiPolygon") + raise ValueError('Geometry must be a Shapely Polygon or MultiPolygon') # if geometry is a MultiPolygon OR a single Polygon whose area exceeds # the max size, get the convex hull around the geometry - if isinstance(geometry, MultiPolygon) or ( - isinstance(geometry, Polygon) and geometry.area > max_query_area_size - ): + if isinstance( + geometry, MultiPolygon) or \ + (isinstance( + geometry, Polygon) and geometry.area > max_query_area_size): geometry = geometry.convex_hull # if geometry area exceeds max size, subdivide it into smaller sub-polygons @@ -410,7 +374,8 @@ def consolidate_subdivide_geometry(geometry, max_query_area_size): return geometry -def quadrat_cut_geometry(geometry, quadrat_width, min_num=3, buffer_amount=1e-9): +def quadrat_cut_geometry(geometry, quadrat_width, min_num=3, + buffer_amount=1e-9): """ Split a Polygon or MultiPolygon up into sub-polygons of a specified size, using quadrats. @@ -435,18 +400,16 @@ def quadrat_cut_geometry(geometry, quadrat_width, min_num=3, buffer_amount=1e-9) # create n evenly spaced points between the min and max x and y bounds lng_max, lat_min, lng_min, lat_max = geometry.bounds - x_num = math.ceil((lng_min - lng_max) / quadrat_width) + 1 - y_num = math.ceil((lat_max - lat_min) / quadrat_width) + 1 + x_num = math.ceil((lng_min-lng_max) / quadrat_width) + 1 + y_num = math.ceil((lat_max-lat_min) / quadrat_width) + 1 x_points = np.linspace(lng_max, lng_min, num=max(x_num, min_num)) y_points = np.linspace(lat_min, lat_max, num=max(y_num, min_num)) # create a quadrat grid of lines at each of the evenly spaced points - vertical_lines = [ - LineString([(x, y_points[0]), (x, y_points[-1])]) for x in x_points - ] - horizont_lines = [ - LineString([(x_points[0], y), (x_points[-1], y)]) for y in y_points - ] + vertical_lines = [LineString([(x, y_points[0]), (x, y_points[-1])]) + for x in x_points] + horizont_lines = [LineString([(x_points[0], y), (x_points[-1], y)]) + for y in y_points] lines = vertical_lines + horizont_lines # buffer each line to distance of the quadrat width divided by 1 billion, @@ -480,11 +443,11 @@ def project_geometry(geometry, crs, to_latlong=False): """ gdf = gpd.GeoDataFrame() gdf.crs = crs - gdf.name = "geometry to project" - gdf["geometry"] = None - gdf.loc[0, "geometry"] = geometry + gdf.name = 'geometry to project' + gdf['geometry'] = None + gdf.loc[0, 'geometry'] = geometry gdf_proj = project_gdf(gdf, to_latlong=to_latlong) - geometry_proj = gdf_proj["geometry"].iloc[0] + geometry_proj = gdf_proj['geometry'].iloc[0] return geometry_proj, gdf_proj.crs @@ -513,7 +476,6 @@ def project_gdf(gdf, to_crs=None, to_latlong=False): gdf_proj : geopandas.GeoDataFrame the projected GeoDataFrame """ - if gdf.crs is None or len(gdf) < 1: raise ValueError("GeoDataFrame must have a valid CRS and cannot be empty") @@ -560,11 +522,13 @@ def process_node(e): node : dict """ - node = {"id": e["id"], "lat": e["lat"], "lon": e["lon"]} + node = {'id': e['id'], + 'lat': e['lat'], + 'lon': e['lon']} - if "tags" in e: - if e["tags"] is not np.nan: - for t, v in list(e["tags"].items()): + if 'tags' in e: + if e['tags'] is not np.nan: + for t, v in list(e['tags'].items()): if t in config.settings.keep_osm_tags: node[t] = v @@ -587,19 +551,19 @@ def process_way(e): waynodes : list of dict """ - way = {"id": e["id"]} + way = {'id': e['id']} - if "tags" in e: - if e["tags"] is not np.nan: - for t, v in list(e["tags"].items()): + if 'tags' in e: + if e['tags'] is not np.nan: + for t, v in list(e['tags'].items()): if t in config.settings.keep_osm_tags: way[t] = v # nodes that make up a way waynodes = [] - for n in e["nodes"]: - waynodes.append({"way_id": e["id"], "node_id": n}) + for n in e['nodes']: + waynodes.append({'way_id': e['id'], 'node_id': n}) return way, waynodes @@ -619,39 +583,32 @@ def parse_network_osm_query(data): nodes, ways, waynodes as a tuple of pandas.DataFrames """ - if len(data["elements"]) == 0: - raise RuntimeError("OSM query results contain no data.") + if len(data['elements']) == 0: + raise RuntimeError('OSM query results contain no data.') nodes = [] ways = [] waynodes = [] - for e in data["elements"]: - if e["type"] == "node": + for e in data['elements']: + if e['type'] == 'node': nodes.append(process_node(e)) - elif e["type"] == "way": + elif e['type'] == 'way': w, wn = process_way(e) ways.append(w) waynodes.extend(wn) - nodes = pd.DataFrame.from_records(nodes, index="id") - ways = pd.DataFrame.from_records(ways, index="id") - waynodes = pd.DataFrame.from_records(waynodes, index="way_id") + nodes = pd.DataFrame.from_records(nodes, index='id') + ways = pd.DataFrame.from_records(ways, index='id') + waynodes = pd.DataFrame.from_records(waynodes, index='way_id') return (nodes, ways, waynodes) -def ways_in_bbox( - lat_min, - lng_min, - lat_max, - lng_max, - network_type, - timeout=180, - memory=None, - max_query_area_size=50 * 1000 * 50 * 1000, - custom_osm_filter=None, -): +def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, + timeout=180, memory=None, + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Get DataFrames of OSM data in a bounding box. @@ -691,18 +648,11 @@ def ways_in_bbox( """ return parse_network_osm_query( - osm_net_download( - lat_max=lat_max, - lat_min=lat_min, - lng_min=lng_min, - lng_max=lng_max, - network_type=network_type, - timeout=timeout, - memory=memory, - max_query_area_size=max_query_area_size, - custom_osm_filter=custom_osm_filter, - ) - ) + osm_net_download(lat_max=lat_max, lat_min=lat_min, lng_min=lng_min, + lng_max=lng_max, network_type=network_type, + timeout=timeout, memory=memory, + max_query_area_size=max_query_area_size, + custom_osm_filter=custom_osm_filter)) def intersection_nodes(waynodes): @@ -753,7 +703,6 @@ def node_pairs(nodes, ways, waynodes, two_way=True): def pairwise(ls): return zip(islice(ls, 0, len(ls)), islice(ls, 1, None)) - intersections = intersection_nodes(waynodes) waymap = waynodes.groupby(level=0, sort=False) pairs = [] @@ -773,11 +722,9 @@ def pairwise(ls): distance = round(gcd(fn.lat, fn.lon, tn.lat, tn.lon), 6) - col_dict = { - "from_id": from_node, - "to_id": to_node, - "distance": distance, - } + col_dict = {'from_id': from_node, + 'to_id': to_node, + 'distance': distance} for tag in config.settings.keep_osm_tags: try: @@ -789,11 +736,9 @@ def pairwise(ls): if not two_way: - col_dict = { - "from_id": to_node, - "to_id": from_node, - "distance": distance, - } + col_dict = {'from_id': to_node, + 'to_id': from_node, + 'distance': distance} for tag in config.settings.keep_osm_tags: try: @@ -805,36 +750,22 @@ def pairwise(ls): pairs = pd.DataFrame.from_records(pairs) if pairs.empty: - raise Exception( - "Query resulted in no connected node pairs. Check " - "your query parameters or bounding box" - ) + raise Exception('Query resulted in no connected node pairs. Check ' + 'your query parameters or bounding box') else: - pairs.index = pd.MultiIndex.from_arrays( - [pairs["from_id"].values, pairs["to_id"].values] - ) - log( - "Edge node pairs completed. Took {:,.2f} seconds".format( - time.time() - start_time - ) - ) + pairs.index = pd.MultiIndex.from_arrays([pairs['from_id'].values, + pairs['to_id'].values]) + log('Edge node pairs completed. Took {:,.2f} seconds' + .format(time.time()-start_time)) return pairs -def network_from_bbox( - lat_min=None, - lng_min=None, - lat_max=None, - lng_max=None, - bbox=None, - network_type="walk", - two_way=True, - timeout=180, - memory=None, - max_query_area_size=50 * 1000 * 50 * 1000, - custom_osm_filter=None, -): +def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, + bbox=None, network_type='walk', two_way=True, + timeout=180, memory=None, + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Make a graph network from a bounding lat/lon box composed of nodes and edges for use in Pandana street network accessibility calculations. @@ -905,53 +836,35 @@ def network_from_bbox( lng_max, lat_min, lng_min, lat_max = bbox - assert lat_min is not None, "lat_min cannot be None" - assert lng_min is not None, "lng_min cannot be None" - assert lat_max is not None, "lat_max cannot be None" - assert lng_max is not None, "lng_max cannot be None" - assert ( - isinstance(lat_min, float) - and isinstance(lng_min, float) - and isinstance(lat_max, float) - and isinstance(lng_max, float) - ), "lat_min, lng_min, lat_max, and lng_max must be floats" + assert lat_min is not None, 'lat_min cannot be None' + assert lng_min is not None, 'lng_min cannot be None' + assert lat_max is not None, 'lat_max cannot be None' + assert lng_max is not None, 'lng_max cannot be None' + assert isinstance(lat_min, float) and isinstance(lng_min, float) and \ + isinstance(lat_max, float) and isinstance(lng_max, float), \ + 'lat_min, lng_min, lat_max, and lng_max must be floats' nodes, ways, waynodes = ways_in_bbox( - lat_min=lat_min, - lng_min=lng_min, - lat_max=lat_max, - lng_max=lng_max, - network_type=network_type, - timeout=timeout, - memory=memory, - max_query_area_size=max_query_area_size, - custom_osm_filter=custom_osm_filter, - ) - log( - "Returning OSM data with {:,} nodes and {:,} ways...".format( - len(nodes), len(ways) - ) - ) + lat_min=lat_min, lng_min=lng_min, lat_max=lat_max, lng_max=lng_max, + network_type=network_type, timeout=timeout, + memory=memory, max_query_area_size=max_query_area_size, + custom_osm_filter=custom_osm_filter) + log('Returning OSM data with {:,} nodes and {:,} ways...' + .format(len(nodes), len(ways))) edgesfinal = node_pairs(nodes, ways, waynodes, two_way=two_way) # make the unique set of nodes that ended up in pairs - node_ids = sorted( - set(edgesfinal["from_id"].unique()).union(set(edgesfinal["to_id"].unique())) - ) + node_ids = sorted(set(edgesfinal['from_id'].unique()) + .union(set(edgesfinal['to_id'].unique()))) nodesfinal = nodes.loc[node_ids] - nodesfinal = nodesfinal[["lon", "lat"]] - nodesfinal.rename(columns={"lon": "x", "lat": "y"}, inplace=True) - nodesfinal["id"] = nodesfinal.index - edgesfinal.rename(columns={"from_id": "from", "to_id": "to"}, inplace=True) - log( - "Returning processed graph with {:,} nodes and {:,} edges...".format( - len(nodesfinal), len(edgesfinal) - ) - ) - log( - "Completed OSM data download and Pandana node and edge table " - "creation in {:,.2f} seconds".format(time.time() - start_time) - ) + nodesfinal = nodesfinal[['lon', 'lat']] + nodesfinal.rename(columns={'lon': 'x', 'lat': 'y'}, inplace=True) + nodesfinal['id'] = nodesfinal.index + edgesfinal.rename(columns={'from_id': 'from', 'to_id': 'to'}, inplace=True) + log('Returning processed graph with {:,} nodes and {:,} edges...' + .format(len(nodesfinal), len(edgesfinal))) + log('Completed OSM data download and Pandana node and edge table ' + 'creation in {:,.2f} seconds'.format(time.time()-start_time)) return nodesfinal, edgesfinal From 3442a3259234a9cbc2638ab201a0425331af734c Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 9 Jul 2020 11:11:33 -0700 Subject: [PATCH 15/25] Removing f-string for py35 --- osmnet/load.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index 0534e33..a4e7ec4 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -497,9 +497,8 @@ def project_gdf(gdf, to_crs=None, to_latlong=False): # calculate UTM zone from avg longitude to define CRS to project to utm_zone = int(math.floor((avg_lng + 180) / 6.0) + 1) - utm_crs = ( - f"+proj=utm +zone={utm_zone} +ellps=WGS84 +datum=WGS84 +units=m +no_defs" - ) + utm_crs = ('+proj=utm +zone={} +ellps=WGS84 +datum=WGS84 +units=m +no_defs' + .format(utm_zone)) # project the GeoDataFrame to the UTM CRS gdf_proj = gdf.to_crs(utm_crs) From 839aaedf4d4490c9b95be5b7888a1e209dfd6099 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 9 Jul 2020 15:10:53 -0700 Subject: [PATCH 16/25] New appveyor script --- appveyor.yml | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index fac07e8..ecf3e7e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,33 +2,24 @@ build: false environment: matrix: - - PYTHON_VERSION: 2.7 - MINICONDA: C:\Miniconda - - PYTHON_VERSION: 3.5 - MINICONDA: C:\Miniconda3 + - PYTHON: 3.8 init: - - "ECHO %PYTHON_VERSION% %MINICONDA%" + - "ECHO %PYTHON%" + +# The goal here is to make sure updates to OSMnet don't introduce any Windows-specific +# runtime errors; the Travis tests running in Linux are more comprehensive. Dependencies +# are installed manually here because the shapely/geopandas stack doesn't install well +# via Pip on Windows. Only using one Python version because AppVeyor builds don't run in +# parallel and can be slow. install: - - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a - - "conda create -q -n test-environment python=%PYTHON_VERSION% numpy pandas pytest pyyaml" - - activate test-environment - - conda install -c conda-forge shapely geopandas - - pip install pycodestyle + - "set PATH=C:\\Miniconda3;C:\\Miniconda3\\Scripts;%PATH%" + - conda config --append channels conda-forge + - "conda create --name test-env python=%PYTHON% pip geopandas numpy pandas pytest requests shapely --yes --quiet" + - activate test-env - pip install . - - pip install wheel + - conda list test_script: - - pycodestyle osmnet - py.test - -after_test: - - "python.exe setup.py bdist_wheel" - -artifacts: - # bdist_wheel puts your built wheel in the dist directory - - path: dist\* From 0008df0dd5b0ce22d9647f99809e45bd2add7d92 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 10 Jul 2020 09:05:27 -0700 Subject: [PATCH 17/25] Better channel resolution --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index ecf3e7e..b586dd1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,9 +15,9 @@ init: install: - "set PATH=C:\\Miniconda3;C:\\Miniconda3\\Scripts;%PATH%" - - conda config --append channels conda-forge - - "conda create --name test-env python=%PYTHON% pip geopandas numpy pandas pytest requests shapely --yes --quiet" + - "conda create --name test-env python=%PYTHON% pip --yes --quiet" - activate test-env + - conda install geopandas numpy pandas pytest requests shapely --channel conda-forge - pip install . - conda list From 1ce4097cda0fd84c611ef9af52ffdd014aad5c38 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 10 Jul 2020 09:14:53 -0700 Subject: [PATCH 18/25] Downgrading appveyor python --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index b586dd1..50b5aec 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,7 +2,7 @@ build: false environment: matrix: - - PYTHON: 3.8 + - PYTHON: 3.6 init: - "ECHO %PYTHON%" From c67236a9275dbf8904d920c379bfb94ae5d945ec Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 10 Jul 2020 11:30:14 -0700 Subject: [PATCH 19/25] Appveyor script fix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 50b5aec..8b4bf7c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,7 +17,7 @@ install: - "set PATH=C:\\Miniconda3;C:\\Miniconda3\\Scripts;%PATH%" - "conda create --name test-env python=%PYTHON% pip --yes --quiet" - activate test-env - - conda install geopandas numpy pandas pytest requests shapely --channel conda-forge + - conda install geopandas numpy pandas pytest requests shapely --channel conda-forge --yes --quiet - pip install . - conda list From 52a2c775083e148a10fe05262cf6c4854cd65bd0 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 10 Jul 2020 11:59:38 -0700 Subject: [PATCH 20/25] Changelog and versions --- HISTORY.rst => CHANGELOG.rst | 8 ++++++++ appveyor.yml | 2 +- docs/source/index.rst | 2 +- setup.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) rename HISTORY.rst => CHANGELOG.rst (88%) diff --git a/HISTORY.rst b/CHANGELOG.rst similarity index 88% rename from HISTORY.rst rename to CHANGELOG.rst index d5959cc..4b599b0 100644 --- a/HISTORY.rst +++ b/CHANGELOG.rst @@ -1,3 +1,11 @@ +v0.1.6 +====== + +2020/7/10 + +* adds support for GeoPandas v0.7 and later +* ends support for Python 2.7 + v0.1.5 ====== diff --git a/appveyor.yml b/appveyor.yml index 8b4bf7c..d88973e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,7 @@ init: # parallel and can be slow. install: - - "set PATH=C:\\Miniconda3;C:\\Miniconda3\\Scripts;%PATH%" + - "set PATH=C:\\Miniconda36-x64;C:\\Miniconda36-x64\\Scripts;%PATH%" - "conda create --name test-env python=%PYTHON% pip --yes --quiet" - activate test-env - conda install geopandas numpy pandas pytest requests shapely --channel conda-forge --yes --quiet diff --git a/docs/source/index.rst b/docs/source/index.rst index 8c31009..779cbe2 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,7 +3,7 @@ OSMnet Tools for the extraction of `OpenStreetMap`_ (OSM) street network data. Intended to be used in tandem with `Pandana`_ and `UrbanAccess`_ libraries to extract street network nodes and edges. -v0.1.6, released July 9, 2020. +v0.1.6, released July 10, 2020. Contents -------- diff --git a/setup.py b/setup.py index 46af67d..d10ffcf 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ install_requires=[ 'geopandas >= 0.7', 'numpy >= 1.10', - 'pandas >= 0.16.0', + 'pandas >= 0.23', 'requests >= 2.9.1', 'shapely >= 1.5' ] From e682f11d7a32e188ff2fd589bc6e47ebcc5e3917 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 10 Jul 2020 12:11:27 -0700 Subject: [PATCH 21/25] Checking simpler conda command --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d88973e..9f74f0a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,9 +15,9 @@ init: install: - "set PATH=C:\\Miniconda36-x64;C:\\Miniconda36-x64\\Scripts;%PATH%" - - "conda create --name test-env python=%PYTHON% pip --yes --quiet" + - conda config --append channels conda-forge + - "conda create --name test-env python=%PYTHON% pip geopandas numpy pandas pytest requests shapely --yes --quiet" - activate test-env - - conda install geopandas numpy pandas pytest requests shapely --channel conda-forge --yes --quiet - pip install . - conda list From 667cfb0b60de400e74cbd24048c7d54a3f9595dc Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 10 Jul 2020 12:17:56 -0700 Subject: [PATCH 22/25] Cleanup --- CHANGELOG.rst | 2 +- MANIFEST.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b599b0..c02038f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,7 +4,7 @@ v0.1.6 2020/7/10 * adds support for GeoPandas v0.7 and later -* ends support for Python 2.7 +* ends support for Python 2.7 and Win32 v0.1.5 ====== diff --git a/MANIFEST.in b/MANIFEST.in index 3b667ca..355a7ea 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ # files to include in the source distribution on pypi (setup and README are included automatically) -include HISTORY.rst +include CHANGELOG.rst include LICENSE.txt From a2d78550bf8f0de9307ee72b0afca3d5fca6ae18 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 10 Jul 2020 12:33:52 -0700 Subject: [PATCH 23/25] Updating doc build --- bin/build_docs.sh | 59 ----------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100755 bin/build_docs.sh diff --git a/bin/build_docs.sh b/bin/build_docs.sh deleted file mode 100755 index 55c0d5c..0000000 --- a/bin/build_docs.sh +++ /dev/null @@ -1,59 +0,0 @@ -#! /usr/bin/env bash - -# Copied from github.com/sympy/sympy -# -# This file automatically deploys changes to http://udst.github.io/osmnet/. -# This will only happen when building a non-pull request build on the master -# branch of Pandana. -# It requires an access token which should be present in .travis.yml file. -# -# Following is the procedure to get the access token: -# -# $ curl -X POST -u -H "Content-Type: application/json" -d\ -# "{\"scopes\":[\"public_repo\"],\"note\":\"token for pushing from travis\"}"\ -# https://api.github.com/authorizations -# -# It'll give you a JSON response having a key called "token". -# -# $ gem install travis -# $ travis encrypt -r sympy/sympy GH_TOKEN= env.global -# -# This will give you an access token("secure"). This helps in creating an -# environment variable named GH_TOKEN while building. -# -# Add this secure code to .travis.yml as described here http://docs.travis-ci.com/user/encryption-keys/ - -# Exit on error -set -e - -ACTUAL_TRAVIS_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER| cut -d'.' -f 2` - -if [ "$TRAVIS_REPO_SLUG" == "UDST/osmnet" ] && \ - [ "$TRAVIS_BRANCH" == "master" ] && \ - [ "$TRAVIS_PULL_REQUEST" == "false" ] && \ - [ "$ACTUAL_TRAVIS_JOB_NUMBER" == "1" ]; then - - echo "Building docs" - cd docs - make clean - make html - - cd ../../ - echo "Setting git attributes" - git config --global user.email "fernandez@urbansim.com" - git config --global user.name "udst-documentator" - - echo "Cloning repository" - git clone --quiet --single-branch --branch=gh-pages https://${GH_TOKEN}@github.com/udst/osmnet.git gh-pages > /dev/null 2>&1 - - cd gh-pages - rm -rf * - cp -R ../osmnet/docs/build/html/* ./ - git add -A . - - git commit -am "Update dev docs after building $TRAVIS_BUILD_NUMBER" - echo "Pushing commit" - git push -fq origin gh-pages - echo "Commit pushed" -#> /dev/null 2>&1 -fi From 1c85c8ef81835439ae98bd774a113caa52084eb7 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 13 Jul 2020 16:12:34 -0700 Subject: [PATCH 24/25] Updated installation instructions --- CHANGELOG.rst | 2 +- docs/source/index.rst | 2 +- docs/source/installation.rst | 79 +++++++++++++++++------------------- 3 files changed, 40 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c02038f..56aa18f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ v0.1.6 ====== -2020/7/10 +2020/7/13 * adds support for GeoPandas v0.7 and later * ends support for Python 2.7 and Win32 diff --git a/docs/source/index.rst b/docs/source/index.rst index 779cbe2..450fba2 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,7 +3,7 @@ OSMnet Tools for the extraction of `OpenStreetMap`_ (OSM) street network data. Intended to be used in tandem with `Pandana`_ and `UrbanAccess`_ libraries to extract street network nodes and edges. -v0.1.6, released July 10, 2020. +v0.1.6, released July 13, 2020. Contents -------- diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 388987c..6257b36 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -1,68 +1,65 @@ Installation -===================== +============ -OSMnet relies on a number of libraries in the scientific Python stack which can be easily installed using the `Anaconda`_ python distribution. +OSMnet is built on top of the Python data science stack, making use of libraries like NumPy and Pandas, plus geospatial packages such as GeoPandas and Shapely. -Dependencies -~~~~~~~~~~~~~~~~~~ +OSMnet v0.1.6 (July 2020) dropped support for Python 2.7 and 32-bit Windows environments. In these older environments, v0.1.5 should install automatically -- but if not, you can get it with ``conda install osmnet=0.1.5 ...`` or ``pip install osmnet==0.1.5``. -* requests >= 2.9.1 -* pandas >= 0.16.0 -* numpy >= 1.10 -* geopandas >= 0.2.1 -* Shapely >= 1.5 -Installation -~~~~~~~~~~~~~~~~~~ - -conda -^^^^^^^^^^^^^ +Conda +^^^^^ -OSMnet is available on conda and can be installed with: +OSMnet is distributed on Conda Forge and can be installed with: ``conda install osmnet --channel conda-forge`` -It is recommended to install via conda and the conda-forge channel especially if you find you are having issues installing some of the spatial dependencies. +This is generally the smoothest installation route. Although OSMnet itself is pure Python code (no compilation needed), the geospatial dependencies often cause installation problems using the default Pip package manager, especially in Windows. You can obtain the Conda package manager by installing the `Anaconda `_ Python distribution. -pip -^^^^^^^^^^^^^ -OSMnet can be installed via PyPI: +Pip +^^^ -``pip install osmnet`` +OSMnet is also distributed on PyPI: -Development Installation -^^^^^^^^^^^^^^^^^^^^^^^^^^ +``pip install osmnet`` -To install use the ``develop`` command rather than ``install``. Make sure you -are using the latest version of the codebase by using git’s ``git pull`` -inside the cloned repository. +If you run into errors related to dependency installation, try (a) setting up a clean environment and installing again, or (b) using Conda instead of Pip. -To install OSMnet follow these steps: -1. Git clone the `OSMnet repo`_ -2. in the cloned directory run: ``python setup.py develop`` or without dependencies: ``python setup.py develop --no-deps`` +Installing from source code +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To update to the latest version: +You can install a development version of OSMnet by cloning the GitHub repository (or a fork or branch) and running this: -Use ``git pull`` inside the cloned repository +``pip install -e .`` -Note for Windows Users when Installing Dependencies -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are a Windows user, dependency installation issues can be minimized by using conda and the conda-forge channel. However, if you find you are still having issues with dependencies such as when importing osmnet you see an error like this: ``ImportError: DLL load failed: The specified module could not be found.`` Most likely one of osmnet's dependencies did not install or compile correctly on your Windows machine. ``geopandas`` requires the dependency package ``fiona`` which requires the dependency package ``gdal``. Windows users could try installing these dependencies via `Christoph Gohlke Windows python wheels`_: `GDAL Windows Wheel`_ and `Fiona Windows Wheel`_. Download the package that matches your Python version and Windows system architecture, then cd into the download directory and install each package for example using: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl`` and -``pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl`` -If you have already installed these packaged via conda or pip, force a reinstall: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall`` and -``pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall`` +Windows troubleshooting +^^^^^^^^^^^^^^^^^^^^^^^ -Current status -~~~~~~~~~~~~~~~~~~ +.. note:: + If you are a Windows user, dependency installation issues can be minimized by using Conda. However, if you find you are still having issues with dependencies -- such as when importing OSMnet you see an error like the one below -- most likely one of OSMnet's dependencies did not install or compile correctly on your machine. GeoPandas requires the dependency package Fiona, which in turn requires the dependency package GDAL. + + .. code-block:: + + ImportError: DLL load failed: The specified module could not be found + + You can try installing these dependencies via `Christoph Gohlke Windows python wheels`_: `GDAL Windows Wheel`_ and `Fiona Windows Wheel`_. Download the package that matches your Python version and Windows system architecture, then ``cd`` into the download directory and install each package like this, changing the file names as appropriate: + + .. code-block:: + + pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl + pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl + + If you have already installed these via Conda or Pip, force a reinstall: + + .. code-block:: + + pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall + pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall -*Forthcoming improvements:* -* Tutorial/demo -.. _Anaconda: http://docs.continuum.io/anaconda/ .. _OSMnet repo: https://github.com/udst/osmnet .. _Christoph Gohlke Windows python wheels: http://www.lfd.uci.edu/~gohlke/pythonlibs/ .. _GDAL Windows Wheel: http://www.lfd.uci.edu/~gohlke/pythonlibs/#gdal From ccad5641e7d9315aceb4c142236d5d4a412951ac Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 14 Jul 2020 15:05:17 -0700 Subject: [PATCH 25/25] Replacing coverage settings --- .coveragerc | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..d2e2dfb --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ +[run] +omit = + osmnet/tests/* + */__init__.py