docker compose changes

.env

@@ -9,6 +9,7 @@ ADMIN_PASS=admin
 # you can change the directory to wherever you want the data to reside on the host
 # MUST be an absolute path
 DATA_DIR=/data/transfer/osm_v2
+TMP_DATA_DIR=/tmp_data/transfer/osm_v2
 
 VALHALLA_URL="http://localhost"
 

conf/valhalla.conf

@@ -16,4 +16,4 @@ redirect_stderr=true
 # stdout_logfile_maxbytes=1MB
 stdout_logfile=/proc/1/fd/1
 stdout_logfile_maxbytes=0
-environment=CONCURRENCY="4",DATA_DIR="/app/data"
+environment=CONCURRENCY="4",DATA_DIR="/app/data",TMP_DATA_DIR="/app/tmp_data"

docker-compose.yml

@@ -4,8 +4,13 @@ volumes:
     driver: local
     driver_opts:
       type: none
-      device: $DATA_DIR  # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file
+      device: $DATA_DIR # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file
       o: bind
+  tmp_packages: # do not change any detail of this volume
+    driver: local
+    driver_opts:
+      type: none
+      device: $TMP_DATA_DIR # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file
 
 # It's important it runs in its own private network, also more secure
 networks:
@@ -50,7 +55,8 @@ services:
       - routing-packager
     volumes:
       - packages:/app/data
-      - $PWD/.docker_env:/app/.env  # Worker needs access to .env file
+      - tmp_packages:/app/tmp_data
+      - $PWD/.docker_env:/app/.env # Worker needs access to .env file
     depends_on:
       - postgis
       - redis
@@ -67,7 +73,8 @@ services:
       - .docker_env
     volumes:
       - packages:/app/data
-      - $PWD/.docker_env:/app/.env  # CLI needs access to .env file
+      - tmp_packages:/app/tmp_data
+      - $PWD/.docker_env:/app/.env # CLI needs access to .env file
       - $PWD/static:/app/static # static file for frontend
     networks:
       - routing-packager

main.py

@@ -20,7 +20,7 @@ async def lifespan(app: FastAPI):
 
     # create the directories
     for provider in Providers:
-        p = SETTINGS.get_data_dir().joinpath(provider.lower())
+        p = SETTINGS.get_tmp_data_dir().joinpath(provider.lower())
         p.mkdir(exist_ok=True)
     SETTINGS.get_output_path().mkdir(exist_ok=True)
     yield

routing_packager_app/config.py

@@ -25,6 +25,7 @@ class BaseSettings(_BaseSettings):
     CORS_ORIGINS: List[str] = ["*"]
 
     DATA_DIR: Path = BASE_DIR.joinpath("data")
+    TMP_DATA_DIR: Path = BASE_DIR.joinpath("tmp_data")
     VALHALLA_URL: str = "http://localhost"
 
     ENABLED_PROVIDERS: list[str] = list(CommaSeparatedStrings("osm"))
@@ -52,7 +53,7 @@ def get_valhalla_path(self, port: int) -> Path:  # pragma: no cover
         Return the path to the OSM Valhalla instances.
         """
         if port in (8002, 8003):
-            p = self.get_data_dir().joinpath(Providers.OSM.lower(), str(port))
+            p = self.get_tmp_data_dir().joinpath(Providers.OSM.lower(), str(port))
             p.mkdir(exist_ok=True, parents=True)
             return p
         raise ValueError(f"{port} is not a valid port for Valhalla.")
@@ -69,6 +70,15 @@ def get_data_dir(self) -> Path:
 
         return data_dir
 
+    def get_tmp_data_dir(self) -> Path:
+        tmp_data_dir = self.TMP_DATA_DIR
+        # if we're inside a docker container, we need to reference the fixed directory instead
+        # Watch out for CI, also runs within docker
+        if os.path.isdir("/app") and not os.getenv("CI", None):  # pragma: no cover
+            tmp_data_dir = Path("/app/tmp_data")
+
+        return tmp_data_dir
+
 
 class ProdSettings(BaseSettings):
     model_config = SettingsConfigDict(case_sensitive=True, env_file=ENV_FILE, extra="ignore")

routing_packager_app/worker.py

@@ -104,7 +104,8 @@ async def create_package(
             raise HTTPException(404, f"No Valhalla tiles in bbox {bbox}")
 
         # zip up the tiles after locking the directory to not be updated right now
-        lock = current_valhalla_dir.joinpath(".lock")
+        out_dir = SETTINGS.get_output_path()
+        lock = out_dir.joinpath(".lock")
         lock.touch(exist_ok=False)
         make_zip(tile_paths, current_valhalla_dir, zip_path)
         lock.unlink(missing_ok=False)

scripts/run_valhalla.sh

@@ -45,12 +45,12 @@ reset_config() {
 
 PORT_8002="8002"
 PORT_8003="8003"
-# $DATA_DIR needs to be defined, either by supervisor or the current shell
-ELEVATION_DIR="$DATA_DIR/elevation"
-VALHALLA_DIR_8002="$DATA_DIR/osm/$PORT_8002"
-VALHALLA_DIR_8003="$DATA_DIR/osm/$PORT_8003"
+# $TMP_DATA_DIR needs to be defined, either by supervisor or the current shell
+ELEVATION_DIR="$TMP_DATA_DIR/elevation"
+VALHALLA_DIR_8002="$TMP_DATA_DIR/osm/$PORT_8002"
+VALHALLA_DIR_8003="$TMP_DATA_DIR/osm/$PORT_8003"
 # TODO: change PBF
-PBF="/app/data/osm/planet-latest.osm.pbf"
+PBF="/app/tmp_data/osm/planet-latest.osm.pbf"
 
 # activate the virtual env so the CLI can do its job in the supervisor env
 . /app/app_venv/bin/activate
@@ -94,7 +94,7 @@ while true; do
     wget -nv https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf -O "$PBF" || exit 1
     # wget -nv https://ftp5.gwdg.de/pub/misc/openstreetmap/download.geofabrik.de/germany-latest.osm.pbf -O "$PBF" || exit 1
     # wget -nv https://download.geofabrik.de/europe/iceland-latest.osm.pbf -O "$PBF" || exit 1
-    # wget -nv https://download.geofabrik.de/europe/andorra-latest.osm.pbf -O "$PBF" || exit 1
+    # wget https://download.geofabrik.de/europe/andorra-latest.osm.pbf -O "$PBF" || exit 1
     UPDATE_OSM="False"
   fi
 
@@ -137,6 +137,8 @@ while true; do
 
   echo "INFO: Downloading elevation to $ELEVATION_DIR..."
   valhalla_build_elevation --from-tiles --decompress -c ${valhalla_config} -v || exit 1
+  # debugging with andorra only:
+  # valhalla_build_elevation --decompress -c ${valhalla_config} -v -b 1,42,2,43 || exit 1
 
   echo "INFO: Enhancing initial tiles with elevation..."
   valhalla_build_tiles -c "${valhalla_config}" -s enhance -e cleanup "$PBF" || exit 1

scripts/update_osm.sh

@@ -20,7 +20,7 @@ usage()
     echo "usage: update_osm.sh --pbf/-p /app/data/osm/planet-latest.osm.pbf"
 }
 
-pbf=/app/data/osm/planet-latest.osm.pbf
+pbf=/app/tmp_data/osm/planet-latest.osm.pbf
 
 # Get the arguments
 while [ "$1" != "" ]; do
@@ -38,7 +38,6 @@ while [ "$1" != "" ]; do
 done
 
 echo "$(date "+%Y-%m-%d %H:%M:%S") Updating ${pbf} with the proxy settings: http_proxy: $http_proxy, https_proxy: $https_proxy"
-
 fn=$(basename "${pbf}")
 pbf_dir=$(dirname "$pbf")
 pbf_name_updated="updated_${fn}"

tests/env

@@ -4,6 +4,7 @@ ADMIN_PASS=admin
 # you can change the directory to wherever you want the data to reside on the host
 # MUST be an absolute path
 DATA_DIR=${PWD}/tests/data
+TMP_DATA_DIR=${PWD}/tests/tmp_data
 
 VALHALLA_SERVER_IP="http://localhost"