Fix quarantined tests - TestCliWebServer (#9598)

GitOrigin-RevId: 48a8316646984e39f57e54cd0b52d49adb0d6cfa

airflow/cli/commands/webserver_command.py

@@ -17,6 +17,7 @@
 
 """Webserver command"""
 import hashlib
+import logging
 import os
 import signal
 import subprocess
@@ -40,6 +41,8 @@
 from airflow.utils.process_utils import check_if_pidfile_process_is_running
 from airflow.www.app import cached_app, create_app
 
+log = logging.getLogger(__name__)
+
 
 class GunicornMonitor(LoggingMixin):
     """
@@ -64,8 +67,8 @@ class GunicornMonitor(LoggingMixin):
     respectively. Gunicorn guarantees that on TTOU workers are terminated
     gracefully and that the oldest worker is terminated.
 
-    :param gunicorn_master_proc:  handle for the main Gunicorn process
-    :param num_workers_expected:  Number of workers to run the Gunicorn web server
+    :param gunicorn_master_pid: PID for the main Gunicorn process
+    :param num_workers_expected: Number of workers to run the Gunicorn web server
     :param master_timeout: Number of seconds the webserver waits before killing gunicorn master that
         doesn't respond
     :param worker_refresh_interval: Number of seconds to wait before refreshing a batch of workers.
@@ -77,15 +80,15 @@ class GunicornMonitor(LoggingMixin):
     """
     def __init__(
         self,
-        gunicorn_master_proc: psutil.Process,
+        gunicorn_master_pid: int,
         num_workers_expected: int,
         master_timeout: int,
         worker_refresh_interval: int,
         worker_refresh_batch_size: int,
         reload_on_plugin_change: bool
     ):
         super().__init__()
-        self.gunicorn_master_proc = gunicorn_master_proc
+        self.gunicorn_master_proc = psutil.Process(gunicorn_master_pid)
         self.num_workers_expected = num_workers_expected
         self.master_timeout = master_timeout
         self.worker_refresh_interval = worker_refresh_interval
@@ -208,8 +211,8 @@ def start(self) -> NoReturn:
                 timeout=self.master_timeout
             )
             while True:
-                if self.gunicorn_master_proc.poll() is not None:
-                    sys.exit(self.gunicorn_master_proc.returncode)
+                if not self.gunicorn_master_proc.is_running():
+                    sys.exit(1)
                 self._check_workers()
                 # Throttle loop
                 sleep(1)
@@ -386,19 +389,20 @@ def webserver(args):
 
         gunicorn_master_proc = None
 
-        def kill_proc(dummy_signum, dummy_frame):  # pylint: disable=unused-argument
+        def kill_proc(signum, _):  # pylint: disable=unused-argument
+            log.info("Received signal: %s. Closing gunicorn.", signum)
             gunicorn_master_proc.terminate()
             gunicorn_master_proc.wait()
             sys.exit(0)
 
-        def monitor_gunicorn(gunicorn_master_proc):
+        def monitor_gunicorn(gunicorn_master_pid: int):
             # Register signal handlers
             signal.signal(signal.SIGINT, kill_proc)
             signal.signal(signal.SIGTERM, kill_proc)
 
             # These run forever until SIG{INT, TERM, KILL, ...} signal is sent
             GunicornMonitor(
-                gunicorn_master_proc=gunicorn_master_proc,
+                gunicorn_master_pid=gunicorn_master_pid,
                 num_workers_expected=num_workers,
                 master_timeout=conf.getint('webserver', 'web_server_master_timeout'),
                 worker_refresh_interval=conf.getint('webserver', 'worker_refresh_interval', fallback=30),
@@ -432,8 +436,8 @@ def monitor_gunicorn(gunicorn_master_proc):
 
                     # Run Gunicorn monitor
                     gunicorn_master_proc = psutil.Process(gunicorn_master_proc_pid)
-                    monitor_gunicorn(gunicorn_master_proc)
+                    monitor_gunicorn(gunicorn_master_proc.pid)
 
         else:
             gunicorn_master_proc = subprocess.Popen(run_args, close_fds=True)
-            monitor_gunicorn(gunicorn_master_proc)
+            monitor_gunicorn(gunicorn_master_proc.pid)

requirements/requirements-python3.6.txt

@@ -72,9 +72,9 @@ beautifulsoup4==4.7.1
 billiard==3.6.3.0
 black==19.10b0
 blinker==1.4
-boto3==1.14.12
+boto3==1.14.14
 boto==2.49.0
-botocore==1.17.12
+botocore==1.17.14
 bowler==0.8.0
 cached-property==1.5.1
 cachetools==4.1.1
@@ -144,7 +144,7 @@ google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
 google-auth==1.18.0
 google-cloud-automl==1.0.1
-google-cloud-bigquery-datatransfer==1.0.0
+google-cloud-bigquery-datatransfer==1.1.0
 google-cloud-bigquery==1.25.0
 google-cloud-bigtable==1.2.1
 google-cloud-container==1.0.1
@@ -206,7 +206,7 @@ jsonpickle==1.4.1
 jsonpointer==2.0
 jsonschema==3.2.0
 junit-xml==1.9
-jupyter-client==6.1.3
+jupyter-client==6.1.5
 jupyter-core==4.6.3
 kombu==4.6.11
 kubernetes==11.0.0
@@ -225,7 +225,7 @@ more-itertools==8.4.0
 moto==1.3.14
 msgpack==1.0.0
 msrest==0.6.17
-msrestazure==0.6.3
+msrestazure==0.6.4
 multi-key-dict==2.0.3
 multidict==4.7.6
 mypy-extensions==0.4.3
@@ -247,7 +247,7 @@ oscrypto==1.2.0
 packaging==20.4
 pandas-gbq==0.13.2
 pandas==1.0.5
-papermill==2.1.1
+papermill==2.1.2
 parameterized==0.7.4
 paramiko==2.7.1
 parso==0.7.0
@@ -299,14 +299,14 @@ pytest-rerunfailures==9.0
 pytest-timeout==1.4.1
 pytest-xdist==1.32.0
 pytest==5.4.3
-python-daemon==2.1.2
+python-daemon==2.2.4
 python-dateutil==2.8.1
 python-editor==1.0.4
 python-http-client==3.2.7
 python-jenkins==1.7.0
 python-jose==3.1.0
 python-nvd3==0.15.0
-python-slugify==4.0.0
+python-slugify==4.0.1
 python3-openid==3.2.0
 pytz==2020.1
 pytzdata==2019.3

requirements/requirements-python3.7.txt

@@ -72,9 +72,9 @@ beautifulsoup4==4.7.1
 billiard==3.6.3.0
 black==19.10b0
 blinker==1.4
-boto3==1.14.12
+boto3==1.14.14
 boto==2.49.0
-botocore==1.17.12
+botocore==1.17.14
 bowler==0.8.0
 cached-property==1.5.1
 cachetools==4.1.1
@@ -143,7 +143,7 @@ google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
 google-auth==1.18.0
 google-cloud-automl==1.0.1
-google-cloud-bigquery-datatransfer==1.0.0
+google-cloud-bigquery-datatransfer==1.1.0
 google-cloud-bigquery==1.25.0
 google-cloud-bigtable==1.2.1
 google-cloud-container==1.0.1
@@ -202,7 +202,7 @@ jsonpickle==1.4.1
 jsonpointer==2.0
 jsonschema==3.2.0
 junit-xml==1.9
-jupyter-client==6.1.3
+jupyter-client==6.1.5
 jupyter-core==4.6.3
 kombu==4.6.11
 kubernetes==11.0.0
@@ -221,7 +221,7 @@ more-itertools==8.4.0
 moto==1.3.14
 msgpack==1.0.0
 msrest==0.6.17
-msrestazure==0.6.3
+msrestazure==0.6.4
 multi-key-dict==2.0.3
 multidict==4.7.6
 mypy-extensions==0.4.3
@@ -243,7 +243,7 @@ oscrypto==1.2.0
 packaging==20.4
 pandas-gbq==0.13.2
 pandas==1.0.5
-papermill==2.1.1
+papermill==2.1.2
 parameterized==0.7.4
 paramiko==2.7.1
 parso==0.7.0
@@ -294,14 +294,14 @@ pytest-rerunfailures==9.0
 pytest-timeout==1.4.1
 pytest-xdist==1.32.0
 pytest==5.4.3
-python-daemon==2.1.2
+python-daemon==2.2.4
 python-dateutil==2.8.1
 python-editor==1.0.4
 python-http-client==3.2.7
 python-jenkins==1.7.0
 python-jose==3.1.0
 python-nvd3==0.15.0
-python-slugify==4.0.0
+python-slugify==4.0.1
 python3-openid==3.2.0
 pytz==2020.1
 pytzdata==2019.3

requirements/requirements-python3.8.txt

@@ -72,9 +72,9 @@ beautifulsoup4==4.7.1
 billiard==3.6.3.0
 black==19.10b0
 blinker==1.4
-boto3==1.14.12
+boto3==1.14.14
 boto==2.49.0
-botocore==1.17.12
+botocore==1.17.14
 bowler==0.8.0
 cached-property==1.5.1
 cachetools==4.1.1
@@ -143,7 +143,7 @@ google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
 google-auth==1.18.0
 google-cloud-automl==1.0.1
-google-cloud-bigquery-datatransfer==1.0.0
+google-cloud-bigquery-datatransfer==1.1.0
 google-cloud-bigquery==1.25.0
 google-cloud-bigtable==1.2.1
 google-cloud-container==1.0.1
@@ -202,7 +202,7 @@ jsonpickle==1.4.1
 jsonpointer==2.0
 jsonschema==3.2.0
 junit-xml==1.9
-jupyter-client==6.1.3
+jupyter-client==6.1.5
 jupyter-core==4.6.3
 kombu==4.6.11
 kubernetes==11.0.0
@@ -221,7 +221,7 @@ more-itertools==8.4.0
 moto==1.3.14
 msgpack==1.0.0
 msrest==0.6.17
-msrestazure==0.6.3
+msrestazure==0.6.4
 multi-key-dict==2.0.3
 multidict==4.7.6
 mypy-extensions==0.4.3
@@ -243,7 +243,7 @@ oscrypto==1.2.0
 packaging==20.4
 pandas-gbq==0.13.2
 pandas==1.0.5
-papermill==2.1.1
+papermill==2.1.2
 parameterized==0.7.4
 paramiko==2.7.1
 parso==0.7.0
@@ -293,14 +293,14 @@ pytest-rerunfailures==9.0
 pytest-timeout==1.4.1
 pytest-xdist==1.32.0
 pytest==5.4.3
-python-daemon==2.1.2
+python-daemon==2.2.4
 python-dateutil==2.8.1
 python-editor==1.0.4
 python-http-client==3.2.7
 python-jenkins==1.7.0
 python-jose==3.1.0
 python-nvd3==0.15.0
-python-slugify==4.0.0
+python-slugify==4.0.1
 python3-openid==3.2.0
 pytz==2020.1
 pytzdata==2019.3

requirements/setup-3.6.md5

@@ -1 +1 @@
-ab047ae7da10b1a5efb746c9c4a403fe  /opt/airflow/setup.py
+e50c855fa7de97b2eca8152389400a5b  /opt/airflow/setup.py

requirements/setup-3.7.md5

@@ -1 +1 @@
-ab047ae7da10b1a5efb746c9c4a403fe  /opt/airflow/setup.py
+e50c855fa7de97b2eca8152389400a5b  /opt/airflow/setup.py

requirements/setup-3.8.md5

@@ -1 +1 @@
-ab047ae7da10b1a5efb746c9c4a403fe  /opt/airflow/setup.py
+e50c855fa7de97b2eca8152389400a5b  /opt/airflow/setup.py

setup.py

@@ -716,7 +716,7 @@ def is_package_excluded(package: str, exclusion_list: List[str]):
     'pep562~=1.0;python_version<"3.7"',
     'psutil>=4.2.0, <6.0.0',
     'pygments>=2.0.1, <3.0',
-    'python-daemon>=2.1.1, <2.2',
+    'python-daemon>=2.1.1',
     'python-dateutil>=2.3, <3',
     'python-nvd3~=0.15.0',
     'python-slugify>=3.0.0,<5.0',