diff --git a/package.json b/package.json index 3882b49..9aad2f6 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@waterthetrees/tree-id": "^1.0.6", "aws-sdk": "^2.1192.0", "colors": "^1.4.0", + "cors": "^2.8.5", "dotenv": "^16.0.1", "express": "^4.18.2", "extract-zip": "^2.0.1", diff --git a/src/config.js b/src/config.js index fad4846..e6d2b8b 100644 --- a/src/config.js +++ b/src/config.js @@ -24,10 +24,13 @@ export const GEOJSON_DIRECTORY = process.env.DATA_DIRECTORY || path.join(DATA_DIRECTORY, "geojson"); export const NORMALIZED_DIRECTORY = process.env.NORMALIZED_DIRECTORY || path.join(DATA_DIRECTORY, "normalized"); - +export const MBTILES_FILEPATH = + process.env.CONCATENATED_FILEPATH || path.join(DATA_DIRECTORY, "mbtiles"); +export const MBTILES_CITIES_FILEPATH = + process.env.CONCATENATED_FILEPATH || path.join(DATA_DIRECTORY, "mbtiles-cities"); export const CONCATENATED_FILEPATH = - process.env.CONCATENATED_FILEPATH || - path.join(DATA_DIRECTORY, "concatenated.geojsons"); + process.env.CONCATENATED_FILEPATH || path.join(DATA_DIRECTORY, "concatenated.geojsons"); + export const TILES_FILEPATH = process.env.TILES_FILEPATH || path.join(DATA_DIRECTORY, "trees.mbtiles"); diff --git a/src/core/sources.js b/src/core/sources.js index 7f8337a..4913303 100644 --- a/src/core/sources.js +++ b/src/core/sources.js @@ -50,8 +50,9 @@ import * as config from "../config.js"; const filenames = await utils.asyncReadDir(config.SOURCES_DIRECTORY); -const promises = filenames.map((name) => { - return import(path.join(config.SOURCES_DIRECTORY, name)); +const promises = filenames.map( async (name) =>{ + const i = await import(path.join(config.SOURCES_DIRECTORY, name)); + return {default: i.default.map(s => {return {...s, filenameSource: name.replace('.js', '')}})}; }); const imports = await Promise.all(promises); @@ -78,13 +79,21 @@ const sources = raw.map((source) => { path: path.join(config.NORMALIZED_DIRECTORY, `${source.idName}.geojsons`), extension: extension, }, + mbtiles: { + path: path.join(config.MBTILES_FILEPATH, `${source.idName}.mbtiles`), + pathOuterZoom: path.join(config.MBTILES_FILEPATH, `${source.idName}.outer-zoom.mbtiles`), + pathMiddleZoom: path.join(config.MBTILES_FILEPATH, `${source.idName}.middle-zoom.mbtiles`), + pathInnerZoom: path.join(config.MBTILES_FILEPATH, `${source.idName}.no-zoom.mbtiles`), + extension: extension, + }, }, }; }); const filterSources = (sourcesArgs) => { if (sourcesArgs) { - return sources.filter(s => sourcesArgs.indexOf(s.idName) != -1); + return sources.filter(s => sourcesArgs.indexOf(s.idName) != -1 + || sourcesArgs.indexOf(s.filenameSource) != -1); } return sources; } diff --git a/src/index.js b/src/index.js index ad8a76f..df9ed10 100644 --- a/src/index.js +++ b/src/index.js @@ -36,7 +36,7 @@ export const runSave = async () => { }; export const runTile = async () => { - await tile.createTiles(); + await tile.createTiles(sources); }; export const runUpload = async () => { diff --git a/src/stages/tile.js b/src/stages/tile.js index d101bec..6742d3d 100644 --- a/src/stages/tile.js +++ b/src/stages/tile.js @@ -1,7 +1,232 @@ import { spawn } from "child_process"; import * as config from "../config.js"; +import path from "path"; +import makeDir from "make-dir"; +import fs from "fs"; +import pLimit from "p-limit"; +import * as utils from "../core/utils.js"; -export const createTiles = async () => { + +export const createTile = async (source) => { + console.log(`Starting for ${source.idName}`); + await makeDir(path.dirname(source.destinations.normalized.path)); + + const normalizedExists = await utils.asyncFileExists( + source.destinations.normalized.path + ); + if (!normalizedExists) { + console.log( + `The expected normalized geojson '${source.destinations.normalizedExists.path}' does not exist. Skipping...` + ); + return `NO FILE for ${source.idName}`; // Early Return + } + + /* + const mbtilesExists = await utils.asyncFileExists( + source.destinations.mbtiles.path + ); + if (mbtilesExists) { + console.log( + `The mbtiles file '${source.destinations.mbtiles.path}' already exists. Skipping...` + ); + return `File already exists for ${source.idName}`; // Early Return + } + */ + + console.log(`Starting outer for ${source.idName}`); + // first, create the outer layer + const childOuterZoom = new Promise((resolve, _) => { + const c = spawn( + "tippecanoe", [ + "--force", + "--cluster-maxzoom=7", + "--maximum-zoom=7", + "-r1", + "-b0", + "--cluster-distance=255", + "-l", + "data", + "-o", + source.destinations.mbtiles.pathOuterZoom, + source.destinations.normalized.path + ] + ); + c.stdout.on('data', (d) => console.log(`stdout: ${d.toString()}`)); + c.stderr.on('data', (d) => console.log(`stdout: ${d.toString()}`)); + c.on("exit", resolve); + }); + + console.log(`Starting middle for ${source.idName}`); + // then the medium ones, still clustered + const childMiddleZoom = new Promise((resolve, _) => { + const c = spawn( + "tippecanoe", [ + "--force", + "--cluster-maxzoom=11", + "--minimum-zoom=8", + "--maximum-zoom=11", + "-r1", + "-b0", + "--cluster-distance=25", + "-l", + "data", + "-o", + source.destinations.mbtiles.pathMiddleZoom, + source.destinations.normalized.path + ] + ); + c.on("exit", resolve); + }); + + console.log(`Starting inner for ${source.idName}`); + // lastly, max zoom when we are close, so use no clustering + const childInnerZoom = new Promise((resolve, _) => { + const c = spawn( + "tippecanoe", [ + "--force", + "--minimum-zoom=12", + "--no-tile-size-limit", + "--drop-densest-as-needed", + "--extend-zooms-if-still-dropping", + "-zg", + "-b0", + "-l", + "data", + "-o", + source.destinations.mbtiles.pathInnerZoom, + source.destinations.normalized.path + ] + ); + c.on("exit", resolve); + }); + + const tileResults = await Promise.allSettled([childMiddleZoom, childInnerZoom, childOuterZoom]); + //const tileResults = await Promise.allSettled([childMiddleZoom, childInnerZoom]); + //const tileResults2 = await Promise.allSettled([childOuterZoom]); + for ( const result of tileResults ) { + if (result.status === 'rejected') { + throw new Error(`Problem running ${source.idName}: ${result}`); + } else { + console.error(result); + } + } + + console.log(`Finished for ${source.idName}`); + + console.log(`Starting combine for ${source.idName}`); + const child = new Promise((resolve, _) => { + const c = spawn( + "tile-join", [ + "--force", + "-o", + source.destinations.mbtiles.path, + source.destinations.mbtiles.pathOuterZoom, + source.destinations.mbtiles.pathMiddleZoom, + source.destinations.mbtiles.pathInnerZoom, + ] + ); + c.on("exit", resolve); + }); + + const resultCombined = await Promise.allSettled([child]); + if (resultCombined.status == 'rejected') { + throw new Error(`Problem running ${source.idName}: ${resultCombined}`); + } + + /* + const deleteResults = await Promise.allSettled([ + fs.unlink(source.destinations.mbtiles.pathOuterZoom), + fs.unlink(source.destinations.mbtiles.pathMiddleZoom), + fs.unlink(source.destinations.mbtiles.pathInnerZoom), + ]); + + if (deleteResults.status == 'rejected') { + throw new Error(`Problem running ${source.idName}: ${deleteResults}`); + } + */ +} + +export const createCombinedTile = async (sources) => { + console.log('Combining the tiles'); + const inner = new Promise((resolve, _) => { + const child = spawn( + "tile-join", [ + "--force", + "-o", + path.join(config.DATA_DIRECTORY, "trees.inner.mbtiles"), + `${config.MBTILES_FILEPATH}/*.no-zoom.mbtiles`, + //...sources.map(s => s.destinations.mbtiles.pathInnerZoom) + ], + { + stdio: ["ignore", process.stdout, process.stderr], + } + ); + child.on("exit", resolve); + }); + + const middle = new Promise((resolve, _) => { + const child = spawn( + "tile-join", [ + "--force", + "-o", + path.join(config.DATA_DIRECTORY, "trees.middle.mbtiles"), + `${config.MBTILES_FILEPATH}/*.middle.mbtiles`, + //...sources.map(s => s.destinations.mbtiles.pathMiddleZoom) + ], + { + stdio: ["ignore", process.stdout, process.stderr], + } + ); + child.on("exit", resolve); + }); + + const outer = new Promise((resolve, _) => { + const child = spawn( + "tile-join", [ + "--force", + "-o", + path.join(config.DATA_DIRECTORY, "trees.outer.mbtiles"), + `${config.MBTILES_FILEPATH}/*.outer.mbtiles`, + //...sources.map(s => s.destinations.mbtiles.pathOuterZoom) + ], + { + stdio: ["ignore", process.stdout, process.stderr], + } + ); + child.on("exit", resolve); + }); + + const all = new Promise((resolve, _) => { + const child = spawn( + "tile-join", [ + "--force", + "-o", + path.join(config.DATA_DIRECTORY, "trees.mbtiles"), + `${config.MBTILES_FILEPATH}/*.mbtiles`, + //...sources.map(s => s.destinations.mbtiles.path) + ], + { + stdio: ["ignore", process.stdout, process.stderr], + } + ); + child.on("exit", resolve); + }); + + console.log('Finished combining the tiles'); + + return Promise.all([inner, middle, outer, all]); +} + + +export const combineOuterZooms = async (sources) => { + /* + * Combine the outer zooms by converting them all back to geojson, + * then combine them all in one call. The idea here is that we can cluster + * if necessary, which happens when two cities are close together + */ +} + +export const createTilesLegacy = async () => { return new Promise((resolve, _) => { const child = spawn( "tippecanoe", @@ -22,3 +247,31 @@ export const createTiles = async () => { child.on("exit", resolve); }); }; + + +export const createTiles = async (list) => { + const limit = pLimit(5); + const promises = list.map((source) => + limit(() => createTile(source)) + ); + const results = await Promise.allSettled(promises); + console.log("Finished creating individual tile files..."); + results.forEach((l) => { + if (l && l.forEach) { + l.forEach(console.log); + } else { + console.log(l); + } + }); + /* + + console.log("Starting to combine individual tile files..."); + const resultCombined = await createCombinedTile( + list.filter(m => ["new_haven", "cambridge", "cary", "bakersfield", "las_vegas", "allentown", "colorado_springs", "marysville"].indexOf(m.idName) == -1) + ); + if (resultCombined.status == 'rejected') { + throw new Error(`Could not combine all tiles: ${resultCombined}`); + } + console.log("Finished combining individual tile files..."); + */ +}; diff --git a/src/tile-server.js b/src/tile-server.js index ab251b6..144ba72 100644 --- a/src/tile-server.js +++ b/src/tile-server.js @@ -1,10 +1,12 @@ import express from "express"; +import cors from "cors"; import * as utils from "./core/utils.js"; import * as tiles from "./core/tiles.js"; import * as config from "./config.js"; // Globals --------------------------------------------------------------------- const app = express(); +app.use(cors()); // Main ------------------------------------------------------------------------ export const run = () => {