From 363142b57e6f1f3a6f177c66652bd7c9be268d84 Mon Sep 17 00:00:00 2001 From: Marcin Szymanski Date: Fri, 26 Jul 2019 09:52:51 +0100 Subject: [PATCH 1/4] [AIRFLOW-5048] Improve display of Kubernetes resources --- airflow/kubernetes/pod.py | 4 ++++ .../test_kubernetes_request_factory.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/airflow/kubernetes/pod.py b/airflow/kubernetes/pod.py index 14c293a1f1576..8a2179d490b0f 100644 --- a/airflow/kubernetes/pod.py +++ b/airflow/kubernetes/pod.py @@ -37,6 +37,10 @@ def has_limits(self): def has_requests(self): return self.request_cpu is not None or self.request_memory is not None + def __str__(self): + return str({'request': {'memory': self.request_memory, 'cpu': self.request_cpu}, + 'limit': {'memory': self.limit_memory}, 'cpu': self.limit_cpu}) + class Port: def __init__( diff --git a/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py b/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py index 312647707ffd2..e8bb5c8b8a605 100644 --- a/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py +++ b/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py @@ -246,6 +246,12 @@ def test_extract_requested_resources(self): KubernetesRequestFactory.extract_resources(pod, self.input_req) self.assertEqual(self.input_req, self.expected) + def test_display_resources(self): + resources_string = str(Resources('1Gi', 1)) + self.assertEqual( + resources_string, + "{'request': {'memory': '1Gi', 'cpu': 1}, 'limit': {'memory': None}, 'cpu': None}") + def test_extract_limits_resources(self): # Test when resources is not empty resources = Resources(limit_memory='2Gi', limit_cpu=2) From fa9c555f157648df177538a951825f2a5a621fef Mon Sep 17 00:00:00 2001 From: Marcin Szymanski Date: Mon, 29 Jul 2019 09:37:25 +0100 Subject: [PATCH 2/4] [AIRFLOW-5048] add gpu limits and fix test conflicts --- airflow/kubernetes/pod.py | 2 +- .../test_kubernetes_request_factory.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/airflow/kubernetes/pod.py b/airflow/kubernetes/pod.py index 730a8f3829630..51e0a3b74aded 100644 --- a/airflow/kubernetes/pod.py +++ b/airflow/kubernetes/pod.py @@ -41,7 +41,7 @@ def has_requests(self): def __str__(self): return str({'request': {'memory': self.request_memory, 'cpu': self.request_cpu}, - 'limit': {'memory': self.limit_memory}, 'cpu': self.limit_cpu}) + 'limit': {'memory': self.limit_memory}, 'cpu': self.limit_cpu, 'gpu': self.limit_gpu}) class Port: diff --git a/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py b/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py index 301b421fce1b5..7057ac597bdd8 100644 --- a/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py +++ b/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py @@ -253,9 +253,7 @@ def test_display_resources(self): resources_string = str(Resources('1Gi', 1)) self.assertEqual( resources_string, - "{'request': {'memory': '1Gi', 'cpu': 1}, 'limit': {'memory': None}, 'cpu': None}") - - def test_extract_limits_resources(self): + "{'request': {'memory': '1Gi', 'cpu': 1}, 'limit': {'memory': None}, 'cpu': None, 'gpu': None}") def test_extract_limits(self): # Test when resources is not empty From 6579ff45d0d7655d6a9d11d49785b9910a455d0c Mon Sep 17 00:00:00 2001 From: Marcin Szymanski Date: Mon, 29 Jul 2019 14:40:16 +0100 Subject: [PATCH 3/4] [AIRFLOW-5048] fix dict ordering for python 3.5 --- airflow/kubernetes/pod.py | 4 ++-- .../test_kubernetes_request_factory.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/kubernetes/pod.py b/airflow/kubernetes/pod.py index 51e0a3b74aded..105d1723a221e 100644 --- a/airflow/kubernetes/pod.py +++ b/airflow/kubernetes/pod.py @@ -40,8 +40,8 @@ def has_requests(self): return self.request_cpu is not None or self.request_memory is not None def __str__(self): - return str({'request': {'memory': self.request_memory, 'cpu': self.request_cpu}, - 'limit': {'memory': self.limit_memory}, 'cpu': self.limit_cpu, 'gpu': self.limit_gpu}) + return str({'limit': {'cpu': self.limit_cpu, 'gpu': self.limit_gpu, 'memory': self.limit_memory}, + 'request': {'cpu': self.request_cpu, 'memory': self.request_memory, }}) class Port: diff --git a/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py b/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py index 7057ac597bdd8..5057857186eae 100644 --- a/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py +++ b/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py @@ -253,7 +253,7 @@ def test_display_resources(self): resources_string = str(Resources('1Gi', 1)) self.assertEqual( resources_string, - "{'request': {'memory': '1Gi', 'cpu': 1}, 'limit': {'memory': None}, 'cpu': None, 'gpu': None}") + "{'limit': {'cpu': None, 'gpu': None, 'memory': None}, 'request': {'cpu': 1, 'memory': '1Gi'}}") def test_extract_limits(self): # Test when resources is not empty From 514350c171987a85a639054c3c9e71053e280363 Mon Sep 17 00:00:00 2001 From: Marcin Szymanski Date: Mon, 29 Jul 2019 16:43:43 +0100 Subject: [PATCH 4/4] [AIRFLOW-5048] avoid Python 3.5/3.6 dict ordering differences --- airflow/kubernetes/pod.py | 5 +++-- .../test_kubernetes_request_factory.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airflow/kubernetes/pod.py b/airflow/kubernetes/pod.py index 105d1723a221e..b799c505f4085 100644 --- a/airflow/kubernetes/pod.py +++ b/airflow/kubernetes/pod.py @@ -40,8 +40,9 @@ def has_requests(self): return self.request_cpu is not None or self.request_memory is not None def __str__(self): - return str({'limit': {'cpu': self.limit_cpu, 'gpu': self.limit_gpu, 'memory': self.limit_memory}, - 'request': {'cpu': self.request_cpu, 'memory': self.request_memory, }}) + return "Request: [cpu: {}, memory: {}], Limit: [cpu: {}, memory: {}, gpu: {}]".format( + self.request_cpu, self.request_memory, self.limit_cpu, self.limit_memory, self.limit_gpu + ) class Port: diff --git a/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py b/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py index 5057857186eae..6786bea0c23cd 100644 --- a/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py +++ b/tests/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py @@ -253,7 +253,7 @@ def test_display_resources(self): resources_string = str(Resources('1Gi', 1)) self.assertEqual( resources_string, - "{'limit': {'cpu': None, 'gpu': None, 'memory': None}, 'request': {'cpu': 1, 'memory': '1Gi'}}") + "Request: [cpu: 1, memory: 1Gi], Limit: [cpu: None, memory: None, gpu: None]") def test_extract_limits(self): # Test when resources is not empty