Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feature/clustering #26

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feature/clustering #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"@waterthetrees/tree-id": "^1.0.6",
"aws-sdk": "^2.1192.0",
"colors": "^1.4.0",
"cors": "^2.8.5",
"dotenv": "^16.0.1",
"express": "^4.18.2",
"extract-zip": "^2.0.1",
Expand Down
9 changes: 6 additions & 3 deletions src/config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@ export const GEOJSON_DIRECTORY =
process.env.DATA_DIRECTORY || path.join(DATA_DIRECTORY, "geojson");
export const NORMALIZED_DIRECTORY =
process.env.NORMALIZED_DIRECTORY || path.join(DATA_DIRECTORY, "normalized");

export const MBTILES_FILEPATH =
process.env.CONCATENATED_FILEPATH || path.join(DATA_DIRECTORY, "mbtiles");
export const MBTILES_CITIES_FILEPATH =
process.env.CONCATENATED_FILEPATH || path.join(DATA_DIRECTORY, "mbtiles-cities");
export const CONCATENATED_FILEPATH =
process.env.CONCATENATED_FILEPATH ||
path.join(DATA_DIRECTORY, "concatenated.geojsons");
process.env.CONCATENATED_FILEPATH || path.join(DATA_DIRECTORY, "concatenated.geojsons");


export const TILES_FILEPATH =
process.env.TILES_FILEPATH || path.join(DATA_DIRECTORY, "trees.mbtiles");
Expand Down
15 changes: 12 additions & 3 deletions src/core/sources.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,9 @@ import * as config from "../config.js";

const filenames = await utils.asyncReadDir(config.SOURCES_DIRECTORY);

const promises = filenames.map((name) => {
return import(path.join(config.SOURCES_DIRECTORY, name));
const promises = filenames.map( async (name) =>{
const i = await import(path.join(config.SOURCES_DIRECTORY, name));
return {default: i.default.map(s => {return {...s, filenameSource: name.replace('.js', '')}})};
});
const imports = await Promise.all(promises);

Expand All @@ -78,13 +79,21 @@ const sources = raw.map((source) => {
path: path.join(config.NORMALIZED_DIRECTORY, `${source.idName}.geojsons`),
extension: extension,
},
mbtiles: {
path: path.join(config.MBTILES_FILEPATH, `${source.idName}.mbtiles`),
pathOuterZoom: path.join(config.MBTILES_FILEPATH, `${source.idName}.outer-zoom.mbtiles`),
pathMiddleZoom: path.join(config.MBTILES_FILEPATH, `${source.idName}.middle-zoom.mbtiles`),
pathInnerZoom: path.join(config.MBTILES_FILEPATH, `${source.idName}.no-zoom.mbtiles`),
extension: extension,
},
},
};
});

const filterSources = (sourcesArgs) => {
if (sourcesArgs) {
return sources.filter(s => sourcesArgs.indexOf(s.idName) != -1);
return sources.filter(s => sourcesArgs.indexOf(s.idName) != -1
|| sourcesArgs.indexOf(s.filenameSource) != -1);
}
return sources;
}
Expand Down
2 changes: 1 addition & 1 deletion src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export const runSave = async () => {
};

export const runTile = async () => {
await tile.createTiles();
await tile.createTiles(sources);
};

export const runUpload = async () => {
Expand Down
255 changes: 254 additions & 1 deletion src/stages/tile.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,232 @@
import { spawn } from "child_process";
import * as config from "../config.js";
import path from "path";
import makeDir from "make-dir";
import fs from "fs";
import pLimit from "p-limit";
import * as utils from "../core/utils.js";

export const createTiles = async () => {

export const createTile = async (source) => {
console.log(`Starting for ${source.idName}`);
await makeDir(path.dirname(source.destinations.normalized.path));

const normalizedExists = await utils.asyncFileExists(
source.destinations.normalized.path
);
if (!normalizedExists) {
console.log(
`The expected normalized geojson '${source.destinations.normalizedExists.path}' does not exist. Skipping...`
);
return `NO FILE for ${source.idName}`; // Early Return
}

/*
const mbtilesExists = await utils.asyncFileExists(
source.destinations.mbtiles.path
);
if (mbtilesExists) {
console.log(
`The mbtiles file '${source.destinations.mbtiles.path}' already exists. Skipping...`
);
return `File already exists for ${source.idName}`; // Early Return
}
*/

console.log(`Starting outer for ${source.idName}`);
// first, create the outer layer
const childOuterZoom = new Promise((resolve, _) => {
const c = spawn(
"tippecanoe", [
"--force",
"--cluster-maxzoom=7",
"--maximum-zoom=7",
"-r1",
"-b0",
"--cluster-distance=255",
"-l",
"data",
"-o",
source.destinations.mbtiles.pathOuterZoom,
source.destinations.normalized.path
]
);
c.stdout.on('data', (d) => console.log(`stdout: ${d.toString()}`));
c.stderr.on('data', (d) => console.log(`stdout: ${d.toString()}`));
c.on("exit", resolve);
});

console.log(`Starting middle for ${source.idName}`);
// then the medium ones, still clustered
const childMiddleZoom = new Promise((resolve, _) => {
const c = spawn(
"tippecanoe", [
"--force",
"--cluster-maxzoom=11",
"--minimum-zoom=8",
"--maximum-zoom=11",
"-r1",
"-b0",
"--cluster-distance=25",
"-l",
"data",
"-o",
source.destinations.mbtiles.pathMiddleZoom,
source.destinations.normalized.path
]
);
c.on("exit", resolve);
});

console.log(`Starting inner for ${source.idName}`);
// lastly, max zoom when we are close, so use no clustering
const childInnerZoom = new Promise((resolve, _) => {
const c = spawn(
"tippecanoe", [
"--force",
"--minimum-zoom=12",
"--no-tile-size-limit",
"--drop-densest-as-needed",
"--extend-zooms-if-still-dropping",
"-zg",
"-b0",
"-l",
"data",
"-o",
source.destinations.mbtiles.pathInnerZoom,
source.destinations.normalized.path
]
);
c.on("exit", resolve);
});

const tileResults = await Promise.allSettled([childMiddleZoom, childInnerZoom, childOuterZoom]);
//const tileResults = await Promise.allSettled([childMiddleZoom, childInnerZoom]);
//const tileResults2 = await Promise.allSettled([childOuterZoom]);
for ( const result of tileResults ) {
if (result.status === 'rejected') {
throw new Error(`Problem running ${source.idName}: ${result}`);
} else {
console.error(result);
}
}

console.log(`Finished for ${source.idName}`);

console.log(`Starting combine for ${source.idName}`);
const child = new Promise((resolve, _) => {
const c = spawn(
"tile-join", [
"--force",
"-o",
source.destinations.mbtiles.path,
source.destinations.mbtiles.pathOuterZoom,
source.destinations.mbtiles.pathMiddleZoom,
source.destinations.mbtiles.pathInnerZoom,
]
);
c.on("exit", resolve);
});

const resultCombined = await Promise.allSettled([child]);
if (resultCombined.status == 'rejected') {
throw new Error(`Problem running ${source.idName}: ${resultCombined}`);
}

/*
const deleteResults = await Promise.allSettled([
fs.unlink(source.destinations.mbtiles.pathOuterZoom),
fs.unlink(source.destinations.mbtiles.pathMiddleZoom),
fs.unlink(source.destinations.mbtiles.pathInnerZoom),
]);

if (deleteResults.status == 'rejected') {
throw new Error(`Problem running ${source.idName}: ${deleteResults}`);
}
*/
}

export const createCombinedTile = async (sources) => {
console.log('Combining the tiles');
const inner = new Promise((resolve, _) => {
const child = spawn(
"tile-join", [
"--force",
"-o",
path.join(config.DATA_DIRECTORY, "trees.inner.mbtiles"),
`${config.MBTILES_FILEPATH}/*.no-zoom.mbtiles`,
//...sources.map(s => s.destinations.mbtiles.pathInnerZoom)
],
{
stdio: ["ignore", process.stdout, process.stderr],
}
);
child.on("exit", resolve);
});

const middle = new Promise((resolve, _) => {
const child = spawn(
"tile-join", [
"--force",
"-o",
path.join(config.DATA_DIRECTORY, "trees.middle.mbtiles"),
`${config.MBTILES_FILEPATH}/*.middle.mbtiles`,
//...sources.map(s => s.destinations.mbtiles.pathMiddleZoom)
],
{
stdio: ["ignore", process.stdout, process.stderr],
}
);
child.on("exit", resolve);
});

const outer = new Promise((resolve, _) => {
const child = spawn(
"tile-join", [
"--force",
"-o",
path.join(config.DATA_DIRECTORY, "trees.outer.mbtiles"),
`${config.MBTILES_FILEPATH}/*.outer.mbtiles`,
//...sources.map(s => s.destinations.mbtiles.pathOuterZoom)
],
{
stdio: ["ignore", process.stdout, process.stderr],
}
);
child.on("exit", resolve);
});

const all = new Promise((resolve, _) => {
const child = spawn(
"tile-join", [
"--force",
"-o",
path.join(config.DATA_DIRECTORY, "trees.mbtiles"),
`${config.MBTILES_FILEPATH}/*.mbtiles`,
//...sources.map(s => s.destinations.mbtiles.path)
],
{
stdio: ["ignore", process.stdout, process.stderr],
}
);
child.on("exit", resolve);
});

console.log('Finished combining the tiles');

return Promise.all([inner, middle, outer, all]);
}


export const combineOuterZooms = async (sources) => {
/*
* Combine the outer zooms by converting them all back to geojson,
* then combine them all in one call. The idea here is that we can cluster
* if necessary, which happens when two cities are close together
*/
}

export const createTilesLegacy = async () => {
return new Promise((resolve, _) => {
const child = spawn(
"tippecanoe",
Expand All @@ -22,3 +247,31 @@ export const createTiles = async () => {
child.on("exit", resolve);
});
};


export const createTiles = async (list) => {
const limit = pLimit(5);
const promises = list.map((source) =>
limit(() => createTile(source))
);
const results = await Promise.allSettled(promises);
console.log("Finished creating individual tile files...");
results.forEach((l) => {
if (l && l.forEach) {
l.forEach(console.log);
} else {
console.log(l);
}
});
/*

console.log("Starting to combine individual tile files...");
const resultCombined = await createCombinedTile(
list.filter(m => ["new_haven", "cambridge", "cary", "bakersfield", "las_vegas", "allentown", "colorado_springs", "marysville"].indexOf(m.idName) == -1)
);
if (resultCombined.status == 'rejected') {
throw new Error(`Could not combine all tiles: ${resultCombined}`);
}
console.log("Finished combining individual tile files...");
*/
};
2 changes: 2 additions & 0 deletions src/tile-server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import express from "express";
import cors from "cors";
import * as utils from "./core/utils.js";
import * as tiles from "./core/tiles.js";
import * as config from "./config.js";

// Globals ---------------------------------------------------------------------
const app = express();
app.use(cors());

// Main ------------------------------------------------------------------------
export const run = () => {
Expand Down