From 031cc7ce4036cc81a1109e9f656c755da0da7cae Mon Sep 17 00:00:00 2001 From: Christian Beiwinkel Date: Fri, 7 Jun 2024 14:01:37 +0200 Subject: [PATCH] docker compose changes --- .env | 1 + conf/valhalla.conf | 2 +- docker-compose.yml | 13 ++++++++++--- main.py | 2 +- routing_packager_app/config.py | 12 +++++++++++- routing_packager_app/worker.py | 3 ++- scripts/run_valhalla.sh | 14 ++++++++------ scripts/update_osm.sh | 3 +-- tests/env | 1 + 9 files changed, 36 insertions(+), 15 deletions(-) diff --git a/.env b/.env index 7c95fc3..e867a75 100644 --- a/.env +++ b/.env @@ -9,6 +9,7 @@ ADMIN_PASS=admin # you can change the directory to wherever you want the data to reside on the host # MUST be an absolute path DATA_DIR=/data/transfer/osm_v2 +TMP_DATA_DIR=/tmp_data/transfer/osm_v2 VALHALLA_URL="http://localhost" diff --git a/conf/valhalla.conf b/conf/valhalla.conf index c0f35a2..04fa27d 100644 --- a/conf/valhalla.conf +++ b/conf/valhalla.conf @@ -16,4 +16,4 @@ redirect_stderr=true # stdout_logfile_maxbytes=1MB stdout_logfile=/proc/1/fd/1 stdout_logfile_maxbytes=0 -environment=CONCURRENCY="4",DATA_DIR="/app/data" +environment=CONCURRENCY="4",DATA_DIR="/app/data",TMP_DATA_DIR="/app/tmp_data" diff --git a/docker-compose.yml b/docker-compose.yml index ca42cc0..454beeb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,13 @@ volumes: driver: local driver_opts: type: none - device: $DATA_DIR # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file + device: $DATA_DIR # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file o: bind + tmp_packages: # do not change any detail of this volume + driver: local + driver_opts: + type: none + device: $TMP_DATA_DIR # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file # It's important it runs in its own private network, also more secure networks: @@ -50,7 +55,8 @@ services: - routing-packager volumes: - packages:/app/data - - $PWD/.docker_env:/app/.env # Worker needs access to .env file + - tmp_packages:/app/tmp_data + - $PWD/.docker_env:/app/.env # Worker needs access to .env file depends_on: - postgis - redis @@ -67,7 +73,8 @@ services: - .docker_env volumes: - packages:/app/data - - $PWD/.docker_env:/app/.env # CLI needs access to .env file + - tmp_packages:/app/tmp_data + - $PWD/.docker_env:/app/.env # CLI needs access to .env file - $PWD/static:/app/static # static file for frontend networks: - routing-packager diff --git a/main.py b/main.py index d67a5d7..892f978 100644 --- a/main.py +++ b/main.py @@ -20,7 +20,7 @@ async def lifespan(app: FastAPI): # create the directories for provider in Providers: - p = SETTINGS.get_data_dir().joinpath(provider.lower()) + p = SETTINGS.get_tmp_data_dir().joinpath(provider.lower()) p.mkdir(exist_ok=True) SETTINGS.get_output_path().mkdir(exist_ok=True) yield diff --git a/routing_packager_app/config.py b/routing_packager_app/config.py index 83ac0a2..0b8c97e 100644 --- a/routing_packager_app/config.py +++ b/routing_packager_app/config.py @@ -25,6 +25,7 @@ class BaseSettings(_BaseSettings): CORS_ORIGINS: List[str] = ["*"] DATA_DIR: Path = BASE_DIR.joinpath("data") + TMP_DATA_DIR: Path = BASE_DIR.joinpath("tmp_data") VALHALLA_URL: str = "http://localhost" ENABLED_PROVIDERS: list[str] = list(CommaSeparatedStrings("osm")) @@ -52,7 +53,7 @@ def get_valhalla_path(self, port: int) -> Path: # pragma: no cover Return the path to the OSM Valhalla instances. """ if port in (8002, 8003): - p = self.get_data_dir().joinpath(Providers.OSM.lower(), str(port)) + p = self.get_tmp_data_dir().joinpath(Providers.OSM.lower(), str(port)) p.mkdir(exist_ok=True, parents=True) return p raise ValueError(f"{port} is not a valid port for Valhalla.") @@ -69,6 +70,15 @@ def get_data_dir(self) -> Path: return data_dir + def get_tmp_data_dir(self) -> Path: + tmp_data_dir = self.TMP_DATA_DIR + # if we're inside a docker container, we need to reference the fixed directory instead + # Watch out for CI, also runs within docker + if os.path.isdir("/app") and not os.getenv("CI", None): # pragma: no cover + tmp_data_dir = Path("/app/tmp_data") + + return tmp_data_dir + class ProdSettings(BaseSettings): model_config = SettingsConfigDict(case_sensitive=True, env_file=ENV_FILE, extra="ignore") diff --git a/routing_packager_app/worker.py b/routing_packager_app/worker.py index b5fe531..b9f844d 100644 --- a/routing_packager_app/worker.py +++ b/routing_packager_app/worker.py @@ -104,7 +104,8 @@ async def create_package( raise HTTPException(404, f"No Valhalla tiles in bbox {bbox}") # zip up the tiles after locking the directory to not be updated right now - lock = current_valhalla_dir.joinpath(".lock") + out_dir = SETTINGS.get_output_path() + lock = out_dir.joinpath(".lock") lock.touch(exist_ok=False) make_zip(tile_paths, current_valhalla_dir, zip_path) lock.unlink(missing_ok=False) diff --git a/scripts/run_valhalla.sh b/scripts/run_valhalla.sh index dec0d6a..08d8a35 100755 --- a/scripts/run_valhalla.sh +++ b/scripts/run_valhalla.sh @@ -45,12 +45,12 @@ reset_config() { PORT_8002="8002" PORT_8003="8003" -# $DATA_DIR needs to be defined, either by supervisor or the current shell -ELEVATION_DIR="$DATA_DIR/elevation" -VALHALLA_DIR_8002="$DATA_DIR/osm/$PORT_8002" -VALHALLA_DIR_8003="$DATA_DIR/osm/$PORT_8003" +# $TMP_DATA_DIR needs to be defined, either by supervisor or the current shell +ELEVATION_DIR="$TMP_DATA_DIR/elevation" +VALHALLA_DIR_8002="$TMP_DATA_DIR/osm/$PORT_8002" +VALHALLA_DIR_8003="$TMP_DATA_DIR/osm/$PORT_8003" # TODO: change PBF -PBF="/app/data/osm/planet-latest.osm.pbf" +PBF="/app/tmp_data/osm/planet-latest.osm.pbf" # activate the virtual env so the CLI can do its job in the supervisor env . /app/app_venv/bin/activate @@ -94,7 +94,7 @@ while true; do wget -nv https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf -O "$PBF" || exit 1 # wget -nv https://ftp5.gwdg.de/pub/misc/openstreetmap/download.geofabrik.de/germany-latest.osm.pbf -O "$PBF" || exit 1 # wget -nv https://download.geofabrik.de/europe/iceland-latest.osm.pbf -O "$PBF" || exit 1 - # wget -nv https://download.geofabrik.de/europe/andorra-latest.osm.pbf -O "$PBF" || exit 1 + # wget https://download.geofabrik.de/europe/andorra-latest.osm.pbf -O "$PBF" || exit 1 UPDATE_OSM="False" fi @@ -137,6 +137,8 @@ while true; do echo "INFO: Downloading elevation to $ELEVATION_DIR..." valhalla_build_elevation --from-tiles --decompress -c ${valhalla_config} -v || exit 1 + # debugging with andorra only: + # valhalla_build_elevation --decompress -c ${valhalla_config} -v -b 1,42,2,43 || exit 1 echo "INFO: Enhancing initial tiles with elevation..." valhalla_build_tiles -c "${valhalla_config}" -s enhance -e cleanup "$PBF" || exit 1 diff --git a/scripts/update_osm.sh b/scripts/update_osm.sh index 0d0d362..6b4e71c 100755 --- a/scripts/update_osm.sh +++ b/scripts/update_osm.sh @@ -20,7 +20,7 @@ usage() echo "usage: update_osm.sh --pbf/-p /app/data/osm/planet-latest.osm.pbf" } -pbf=/app/data/osm/planet-latest.osm.pbf +pbf=/app/tmp_data/osm/planet-latest.osm.pbf # Get the arguments while [ "$1" != "" ]; do @@ -38,7 +38,6 @@ while [ "$1" != "" ]; do done echo "$(date "+%Y-%m-%d %H:%M:%S") Updating ${pbf} with the proxy settings: http_proxy: $http_proxy, https_proxy: $https_proxy" - fn=$(basename "${pbf}") pbf_dir=$(dirname "$pbf") pbf_name_updated="updated_${fn}" diff --git a/tests/env b/tests/env index 85e31ba..fbd7cec 100644 --- a/tests/env +++ b/tests/env @@ -4,6 +4,7 @@ ADMIN_PASS=admin # you can change the directory to wherever you want the data to reside on the host # MUST be an absolute path DATA_DIR=${PWD}/tests/data +TMP_DATA_DIR=${PWD}/tests/tmp_data VALHALLA_SERVER_IP="http://localhost"