From 73a2df630f046f3d8e0b8f17e88ea4a53bd8fee6 Mon Sep 17 00:00:00 2001 From: raphaelauv Date: Sat, 17 Jul 2021 18:17:31 +0200 Subject: [PATCH 1/3] [FIX] Docker provider - retry docker in docker --- airflow/providers/docker/operators/docker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/providers/docker/operators/docker.py b/airflow/providers/docker/operators/docker.py index e167113faaa789..5387e48d1623ba 100644 --- a/airflow/providers/docker/operators/docker.py +++ b/airflow/providers/docker/operators/docker.py @@ -247,12 +247,12 @@ def _run_image(self) -> Optional[str]: if not self.cli: raise Exception("The 'cli' should be initialized before!") if self.mount_tmp_dir: - with TemporaryDirectory(prefix='airflowtmp', dir=self.host_tmp_dir) as host_tmp_dir: - tmp_mount = Mount(self.tmp_dir, host_tmp_dir, "bind") + with TemporaryDirectory(prefix='airflowtmp', dir=self.host_tmp_dir) as host_tmp_dir_generated: + tmp_mount = Mount(self.tmp_dir, host_tmp_dir_generated, "bind") try: return self._run_image_with_mounts(self.mounts + [tmp_mount], add_tmp_variable=True) except APIError as e: - if self.host_tmp_dir in str(e): + if host_tmp_dir_generated in str(e): self.log.warning( "Using remote engine or docker-in-docker and mounting temporary " "volume from host is not supported. Falling back to " From 183ecf83beec024c56ea92046c6da6d7834e9d9e Mon Sep 17 00:00:00 2001 From: raphaelauv Date: Sat, 17 Jul 2021 20:46:44 +0200 Subject: [PATCH 2/3] fix test --- tests/providers/docker/operators/test_docker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/providers/docker/operators/test_docker.py b/tests/providers/docker/operators/test_docker.py index bb4c107e937208..db69ce4f132500 100644 --- a/tests/providers/docker/operators/test_docker.py +++ b/tests/providers/docker/operators/test_docker.py @@ -35,11 +35,13 @@ pass +TEMPDIR_MOCK_RETURN_VALUE = '/mkdtemp' + class TestDockerOperator(unittest.TestCase): def setUp(self): self.tempdir_patcher = mock.patch('airflow.providers.docker.operators.docker.TemporaryDirectory') self.tempdir_mock = self.tempdir_patcher.start() - self.tempdir_mock.return_value.__enter__.return_value = '/mkdtemp' + self.tempdir_mock.return_value.__enter__.return_value = TEMPDIR_MOCK_RETURN_VALUE self.client_mock = mock.Mock(spec=APIClient) self.client_mock.create_container.return_value = {'Id': 'some_id'} @@ -185,7 +187,7 @@ def test_execute_no_temp_dir(self): def test_execute_fallback_temp_dir(self): self.client_mock.create_container.side_effect = [ - APIError(message="wrong path: " + "/host/airflow"), + APIError(message="wrong path: " +TEMPDIR_MOCK_RETURN_VALUE ), {'Id': 'some_id'}, ] operator = DockerOperator( From ad77c799f04a0be17e54ac422535e34affb86008 Mon Sep 17 00:00:00 2001 From: raphaelauv Date: Sat, 17 Jul 2021 21:38:16 +0200 Subject: [PATCH 3/3] fix linter --- tests/providers/docker/operators/test_docker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/providers/docker/operators/test_docker.py b/tests/providers/docker/operators/test_docker.py index db69ce4f132500..78bb9680299a01 100644 --- a/tests/providers/docker/operators/test_docker.py +++ b/tests/providers/docker/operators/test_docker.py @@ -37,6 +37,7 @@ TEMPDIR_MOCK_RETURN_VALUE = '/mkdtemp' + class TestDockerOperator(unittest.TestCase): def setUp(self): self.tempdir_patcher = mock.patch('airflow.providers.docker.operators.docker.TemporaryDirectory') @@ -187,7 +188,7 @@ def test_execute_no_temp_dir(self): def test_execute_fallback_temp_dir(self): self.client_mock.create_container.side_effect = [ - APIError(message="wrong path: " +TEMPDIR_MOCK_RETURN_VALUE ), + APIError(message="wrong path: " + TEMPDIR_MOCK_RETURN_VALUE), {'Id': 'some_id'}, ] operator = DockerOperator(