From 49b2a8d1109ca215d1c73cc453760b733bc772ff Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Thu, 2 Jun 2022 11:34:22 -0700 Subject: [PATCH 1/2] test --- dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 05bcc11d6cb87..30960e8809295 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -121,7 +121,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: for (src, dst) in NECESSARY_HOST_VOLUMES: if (AIRFLOW_SOURCES_ROOT / src).exists(): extra_docker_flags.extend( - ["--mount", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}'] + ["--volume", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}'] ) extra_docker_flags.extend( ['--mount', "type=volume,src=docker-compose_mypy-cache-volume,dst=/opt/airflow/.mypy_cache"] From b978af2e2c958dba9ffb28a40986e61e2d32561b Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Thu, 2 Jun 2022 11:43:22 -0700 Subject: [PATCH 2/2] Breeze must create `hooks\` and `dags\` directories for bind mounts - Now that breeze uses --mount instead of --volume (the former of which does not create missing mount dirs like the latter does see docs here: https://docs.docker.com/storage/bind-mounts/#differences-between--v-and---mount-behavior) we need to create these directories explicitly. --- dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 2 +- dev/breeze/src/airflow_breeze/utils/path_utils.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 30960e8809295..05bcc11d6cb87 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -121,7 +121,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: for (src, dst) in NECESSARY_HOST_VOLUMES: if (AIRFLOW_SOURCES_ROOT / src).exists(): extra_docker_flags.extend( - ["--volume", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}'] + ["--mount", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}'] ) extra_docker_flags.extend( ['--mount', "type=volume,src=docker-compose_mypy-cache-volume,dst=/opt/airflow/.mypy_cache"] diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 759c3e09a7d6a..bd84553731c46 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -237,7 +237,9 @@ def find_airflow_sources_root_to_operate_on() -> Path: AIRFLOW_SOURCES_ROOT = find_airflow_sources_root_to_operate_on().resolve() BUILD_CACHE_DIR = AIRFLOW_SOURCES_ROOT / '.build' +DAGS_DIR = AIRFLOW_SOURCES_ROOT / 'dags' FILES_DIR = AIRFLOW_SOURCES_ROOT / 'files' +HOOKS_DIR = AIRFLOW_SOURCES_ROOT / 'hooks' MSSQL_DATA_VOLUME = AIRFLOW_SOURCES_ROOT / 'tmp_mssql_volume' KUBE_DIR = AIRFLOW_SOURCES_ROOT / ".kube" LOGS_DIR = AIRFLOW_SOURCES_ROOT / 'logs' @@ -255,7 +257,9 @@ def create_directories_and_files() -> None: Checks if setup has been updates since last time and proposes to upgrade if so. """ BUILD_CACHE_DIR.mkdir(parents=True, exist_ok=True) + DAGS_DIR.mkdir(parents=True, exist_ok=True) FILES_DIR.mkdir(parents=True, exist_ok=True) + HOOKS_DIR.mkdir(parents=True, exist_ok=True) MSSQL_DATA_VOLUME.mkdir(parents=True, exist_ok=True) KUBE_DIR.mkdir(parents=True, exist_ok=True) LOGS_DIR.mkdir(parents=True, exist_ok=True)