Wait for asset compilation to finish before starting airflow in Breeze (

#28575) Asset compilation is performed in background and especially if it is run for the first time it might take some time. At the same time database intialization is done so usually it is not a problem but it some cases database might be initialized faster and webserver would start without assets compiled leading to nasty output. This change adds waiting for the compilation - tmux will not split screens and run webserver before it is done.
apache · Dec 24, 2022 · 979a72a · 979a72a
1 parent c0a7bf2
commit 979a72a
Show file tree

Hide file tree

Showing 11 changed files with 113 additions and 17 deletions.
diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -598,6 +598,35 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}
 
 : "${AIRFLOW_SOURCES:?"ERROR: AIRFLOW_SOURCES not set !!!!"}"
 
+function wait_for_asset_compilation() {
+    if [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; then
+        echo
+        echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}"
+        echo
+        local counter=0
+        while [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; do
+            echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}"
+            sleep 1
+            ((counter=counter+1))
+            if [[ ${counter} == "30" ]]; then
+                echo
+                echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}"
+                echo """
+If it does not complete soon, you might want to stop it and remove file lock:
+   * press Ctrl-C
+   * run 'rm ${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock'
+"""
+            fi
+            if [[ ${counter} == "60" ]]; then
+                echo
+                echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}"
+                echo
+                exit 1
+            fi
+        done
+    fi
+}
+
 if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
 
     if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
@@ -785,6 +814,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
     if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then
         export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
         export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
+        wait_for_asset_compilation
         # shellcheck source=scripts/in_container/bin/run_tmux
         exec run_tmux
     fi

diff --git a/dev/breeze/README.md b/dev/breeze/README.md
@@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT.
 
 ---------------------------------------------------------------------------------------------------------
 
-Package config hash: f28f0d555b81a0f48d6b29b3cf8bba132b8c6a8f3d290a25ad4fd62019a9adbf86c0dc913c474e23ae110f3f433db0214bf46b21000f0d2bdd0884134923ae91
+Package config hash: 303fcf07d47a537ae4bf82f30328bb6a30440b11fc8e9dfa7f5188398c18103c4927129b245420ab7fd65e9bc16d359528f74e2d616f8b9f92c15e0e9a9b052c
 
 ---------------------------------------------------------------------------------------------------------
diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg
@@ -55,6 +55,7 @@ packages = find:
 install_requires =
     cached_property>=1.5.0;python_version<="3.7"
     click
+    filelock
     inputimeout
     importlib-metadata>=4.4; python_version < "3.8"
     jinja2

diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -74,6 +74,7 @@
 VOLUMES_FOR_SELECTED_MOUNTS = [
     (".bash_aliases", "/root/.bash_aliases"),
     (".bash_history", "/root/.bash_history"),
+    (".build", "/opt/airflow/.build"),
     (".coveragerc", "/opt/airflow/.coveragerc"),
     (".dockerignore", "/opt/airflow/.dockerignore"),
     (".flake8", "/opt/airflow/.flake8"),

diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -263,6 +263,9 @@ def find_airflow_sources_root_to_operate_on() -> Path:
 
 AIRFLOW_SOURCES_ROOT = find_airflow_sources_root_to_operate_on().resolve()
 BUILD_CACHE_DIR = AIRFLOW_SOURCES_ROOT / ".build"
+WWW_CACHE_DIR = BUILD_CACHE_DIR / "www"
+WWW_ASSET_COMPILE_LOCK = WWW_CACHE_DIR / ".asset_compile.lock"
+WWW_ASSET_OUT_FILE = WWW_CACHE_DIR / "asset_compile.out"
 DAGS_DIR = AIRFLOW_SOURCES_ROOT / "dags"
 FILES_DIR = AIRFLOW_SOURCES_ROOT / "files"
 HOOKS_DIR = AIRFLOW_SOURCES_ROOT / "hooks"

diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -36,7 +36,7 @@
 from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY
 from airflow_breeze.utils.ci_group import ci_group
 from airflow_breeze.utils.console import Output, get_console
-from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
+from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, WWW_ASSET_COMPILE_LOCK, WWW_ASSET_OUT_FILE
 from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
 
 RunCommandResult = Union[subprocess.CompletedProcess, subprocess.CalledProcessError]
@@ -393,16 +393,43 @@ def get_ci_image_for_pre_commits() -> str:
     return airflow_image
 
 
-def _run_compile_internally(command_to_execute: list[str]):
+def _run_compile_internally(command_to_execute: list[str], dev: bool) -> RunCommandResult:
+    from filelock import SoftFileLock, Timeout
+
     env = os.environ.copy()
-    compile_www_assets_result = run_command(
-        command_to_execute,
-        check=False,
-        no_output_dump_on_exception=True,
-        text=True,
-        env=env,
-    )
-    return compile_www_assets_result
+    if dev:
+        return run_command(
+            command_to_execute,
+            check=False,
+            no_output_dump_on_exception=True,
+            text=True,
+            env=env,
+        )
+    else:
+        WWW_ASSET_COMPILE_LOCK.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            WWW_ASSET_COMPILE_LOCK.unlink()
+        except FileNotFoundError:
+            pass
+        try:
+            with SoftFileLock(WWW_ASSET_COMPILE_LOCK, timeout=5):
+                with open(WWW_ASSET_OUT_FILE, "w") as f:
+                    return run_command(
+                        command_to_execute,
+                        check=False,
+                        no_output_dump_on_exception=True,
+                        text=True,
+                        env=env,
+                        stderr=subprocess.STDOUT,
+                        stdout=f,
+                    )
+        except Timeout:
+            get_console().print("[error]Another asset compilation is running. Exiting[/]\n")
+            get_console().print("[warning]If you are sure there is no other compilation,[/]")
+            get_console().print("[warning]Remove the lock file and re-run compilation:[/]")
+            get_console().print(WWW_ASSET_COMPILE_LOCK)
+            get_console().print()
+            sys.exit(1)
 
 
 def run_compile_www_assets(
@@ -425,9 +452,11 @@ def run_compile_www_assets(
         "manual",
         "compile-www-assets-dev" if dev else "compile-www-assets",
         "--all-files",
+        "--verbose",
     ]
+    get_console().print(f"[info] The output of the asset compilation is stored in: [/]{WWW_ASSET_OUT_FILE}\n")
     if run_in_background:
-        thread = Thread(daemon=True, target=_run_compile_internally, args=(command_to_execute,))
+        thread = Thread(daemon=True, target=_run_compile_internally, args=(command_to_execute, dev))
         thread.start()
     else:
-        return _run_compile_internally(command_to_execute)
+        return _run_compile_internally(command_to_execute, dev)
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
@@ -33,6 +33,9 @@ services:
       - type: bind
         source: ../../../.bash_history
         target: /root/.bash_history
+      - type: bind
+        source: ../../../.build
+        target: /opt/airflow/.build
       - type: bind
         source: ../../../.coveragerc
         target: /opt/airflow/.coveragerc

diff --git a/scripts/ci/pre_commit/pre_commit_compile_www_assets.py b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py
@@ -26,7 +26,7 @@
 from common_precommit_utils import get_directory_hash  # isort: skip # noqa
 
 AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve()
-WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www_dir_hash.txt"
+WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www" / "hash.txt"
 
 if __name__ not in ("__main__", "__mp_main__"):
     raise SystemExit(
@@ -42,8 +42,8 @@
     if new_hash == old_hash:
         print("The WWW directory has not changed! Skip regeneration.")
         sys.exit(0)
-    WWW_HASH_FILE.write_text(new_hash)
     env = os.environ.copy()
     env["FORCE_COLOR"] = "true"
     subprocess.check_call(["yarn", "install", "--frozen-lockfile"], cwd=str(www_directory))
     subprocess.check_call(["yarn", "run", "build"], cwd=str(www_directory), env=env)
+    WWW_HASH_FILE.write_text(new_hash)
diff --git a/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py b/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py
@@ -28,7 +28,7 @@
     )
 
 AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve()
-WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www_dir_hash.txt"
+WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www" / "hash.txt"
 
 if __name__ == "__main__":
     www_directory = Path("airflow") / "www"

diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
@@ -45,6 +45,35 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}
 
 : "${AIRFLOW_SOURCES:?"ERROR: AIRFLOW_SOURCES not set !!!!"}"
 
+function wait_for_asset_compilation() {
+    if [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; then
+        echo
+        echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}"
+        echo
+        local counter=0
+        while [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; do
+            echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}"
+            sleep 1
+            ((counter=counter+1))
+            if [[ ${counter} == "30" ]]; then
+                echo
+                echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}"
+                echo """
+If it does not complete soon, you might want to stop it and remove file lock:
+   * press Ctrl-C
+   * run 'rm ${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock'
+"""
+            fi
+            if [[ ${counter} == "60" ]]; then
+                echo
+                echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}"
+                echo
+                exit 1
+            fi
+        done
+    fi
+}
+
 if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
 
     if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
@@ -232,6 +261,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
     if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then
         export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
         export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
+        wait_for_asset_compilation
         # shellcheck source=scripts/in_container/bin/run_tmux
         exec run_tmux
     fi

diff --git a/scripts/in_container/bin/run_tmux b/scripts/in_container/bin/run_tmux
@@ -15,7 +15,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
 if [ ! -e /usr/local/bin/stop_airflow ]; then
     ln -s "/opt/airflow/scripts/in_container/stop_tmux_airflow.sh" /usr/local/bin/stop_airflow || true
 fi