From 481192c5468c908f28f77ce697cae13350409397 Mon Sep 17 00:00:00 2001 From: rmkeezer Date: Mon, 22 Mar 2021 09:39:50 -0500 Subject: [PATCH] feat(python): add max retry configuration for python requests session --- .gitignore | 1 + .secrets.baseline | 24 ++++++++--- ibm_cloud_sdk_core/base_service.py | 41 ++++++++++++++++-- resources/ibm-credentials-retry.env | 6 +++ test/test_base_service.py | 67 +++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 resources/ibm-credentials-retry.env diff --git a/.gitignore b/.gitignore index b801dd4..4d1e67c 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ venv*/ python3/ *.env +!resources/*.env .sfdx/tools/apex.db .pytest_cache/ diff --git a/.secrets.baseline b/.secrets.baseline index c92ab65..3a73f77 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "package-lock.json|^.secrets.baseline$", "lines": null }, - "generated_at": "2021-02-24T18:40:26Z", + "generated_at": "2021-03-18T14:18:04Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -132,6 +132,16 @@ "verified_result": null } ], + "resources/ibm-credentials-retry.env": [ + { + "hashed_secret": "ce49dd46e23153d6593eccd68534b9f1465d5bbd", + "is_secret": false, + "is_verified": false, + "line_number": 1, + "type": "Secret Keyword", + "verified_result": null + } + ], "resources/ibm-credentials.env": [ { "hashed_secret": "b9cad336062c0dc3bb30145b1a6697fccfe755a6", @@ -219,7 +229,7 @@ "hashed_secret": "da2f27d2c57a0e1ed2dc3a34b4ef02faf2f7a4c2", "is_secret": false, "is_verified": false, - "line_number": 27, + "line_number": 29, "type": "Hex High Entropy String", "verified_result": null }, @@ -227,7 +237,7 @@ "hashed_secret": "b3f00e146afe19aab0069029b7fb3926ad756d26", "is_secret": false, "is_verified": false, - "line_number": 99, + "line_number": 107, "type": "Hex High Entropy String", "verified_result": null }, @@ -235,7 +245,7 @@ "hashed_secret": "62cdb7020ff920e5aa642c3d4066950dd1f01f4d", "is_secret": false, "is_verified": false, - "line_number": 159, + "line_number": 170, "type": "Secret Keyword", "verified_result": null } @@ -255,7 +265,7 @@ "hashed_secret": "34a0a47a51d5bf739df0214450385e29ee7e9847", "is_secret": false, "is_verified": false, - "line_number": 253, + "line_number": 353, "type": "Secret Keyword", "verified_result": null }, @@ -263,7 +273,7 @@ "hashed_secret": "2863fa4b5510c46afc2bd2998dfbc0cf3d6df032", "is_secret": false, "is_verified": false, - "line_number": 329, + "line_number": 429, "type": "Secret Keyword", "verified_result": null }, @@ -271,7 +281,7 @@ "hashed_secret": "b9cad336062c0dc3bb30145b1a6697fccfe755a6", "is_secret": false, "is_verified": false, - "line_number": 390, + "line_number": 490, "type": "Secret Keyword", "verified_result": null } diff --git a/ibm_cloud_sdk_core/base_service.py b/ibm_cloud_sdk_core/base_service.py index 199f20b..afc8275 100644 --- a/ibm_cloud_sdk_core/base_service.py +++ b/ibm_cloud_sdk_core/base_service.py @@ -24,7 +24,9 @@ from typing import Dict, List, Optional, Tuple, Union import requests +from requests.adapters import HTTPAdapter from requests.structures import CaseInsensitiveDict +from urllib3.util.retry import Retry from ibm_cloud_sdk_core.authenticators import Authenticator from .version import __version__ from .utils import (has_bad_first_or_last_char, remove_null_values, @@ -88,11 +90,35 @@ def __init__(self, self.default_headers = None self.enable_gzip_compression = enable_gzip_compression self._set_user_agent_header(self._build_user_agent()) + self.retry_config = None + self.http_adapter = HTTPAdapter() if not self.authenticator: raise ValueError('authenticator must be provided') if not isinstance(self.authenticator, Authenticator): raise ValueError('authenticator should be of type Authenticator') + def enable_retries(self, max_retries: int = 4, retry_interval: float = 0.1) -> None: + """Setup http_client with retry_config and http_adapter""" + self.retry_config = Retry( + total = max_retries, + backoff_factor = retry_interval, + # List of HTTP status codes to retry on in addition to Timeout/Connection Errors + status_forcelist = [429, 500, 502, 503, 504], + # List of HTTP methods to retry on + # Omitting this will default to all methods except POST + allowed_methods=['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'] + ) + self.http_adapter = HTTPAdapter(max_retries=self.retry_config) + self.http_client.mount('http://', self.http_adapter) + self.http_client.mount('https://', self.http_adapter) + + def disable_retries(self): + """Remove retry config from http_adapter""" + self.retry_config = None + self.http_adapter = HTTPAdapter() + self.http_client.mount('http://', self.http_adapter) + self.http_client.mount('https://', self.http_adapter) + @staticmethod def _get_system_info() -> str: return '{0} {1} {2}'.format( @@ -126,10 +152,19 @@ def configure_service(self, service_name: str) -> None: if config.get('URL'): self.set_service_url(config.get('URL')) if config.get('DISABLE_SSL'): - self.set_disable_ssl_verification(bool(config.get('DISABLE_SSL'))) - if config.get('ENABLE_GZIP') is not None: + self.set_disable_ssl_verification( + config.get('DISABLE_SSL').lower() == 'true') + if config.get('ENABLE_GZIP'): self.set_enable_gzip_compression( - config.get('ENABLE_GZIP') == 'True') + config.get('ENABLE_GZIP').lower() == 'true') + if config.get('ENABLE_RETRIES'): + if config.get('ENABLE_RETRIES').lower() == 'true': + kwargs = {} + if config.get('MAX_RETRIES'): + kwargs["max_retries"] = int(config.get('MAX_RETRIES')) + if config.get('RETRY_INTERVAL'): + kwargs["retry_interval"] = float(config.get('RETRY_INTERVAL')) + self.enable_retries(**kwargs) def _set_user_agent_header(self, user_agent_string: str) -> None: self.user_agent_header = {'User-Agent': user_agent_string} diff --git a/resources/ibm-credentials-retry.env b/resources/ibm-credentials-retry.env new file mode 100644 index 0000000..37f7aa1 --- /dev/null +++ b/resources/ibm-credentials-retry.env @@ -0,0 +1,6 @@ +INCLUDE_EXTERNAL_CONFIG_APIKEY=mockkey +INCLUDE_EXTERNAL_CONFIG_AUTH_TYPE=iam +INCLUDE_EXTERNAL_CONFIG_URL=https://mockurl +INCLUDE_EXTERNAL_CONFIG_MAX_RETRIES=3 +INCLUDE_EXTERNAL_CONFIG_RETRY_INTERVAL=0.2 +INCLUDE_EXTERNAL_CONFIG_ENABLE_RETRIES=true \ No newline at end of file diff --git a/test/test_base_service.py b/test/test_base_service.py index 4eb653e..553b062 100644 --- a/test/test_base_service.py +++ b/test/test_base_service.py @@ -11,6 +11,7 @@ import responses import requests import jwt +from urllib3.exceptions import ConnectTimeoutError, MaxRetryError from ibm_cloud_sdk_core import BaseService, DetailedResponse from ibm_cloud_sdk_core import ApiException from ibm_cloud_sdk_core import CP4DTokenManager @@ -553,6 +554,72 @@ def test_gzip_compression_external(): assert prepped['data'] == gzip.compress(b'{"foo": "bar"}') assert prepped['headers'].get('content-encoding') == 'gzip' +def test_retry_config_default(): + service = BaseService(service_url='https://mockurl/', authenticator=NoAuthAuthenticator()) + service.enable_retries() + assert service.retry_config.total == 4 + assert service.retry_config.backoff_factor == 0.1 + assert service.http_client.get_adapter('https://').max_retries.total == 4 + + # Ensure retries fail after 4 retries + error = ConnectTimeoutError() + retry = service.http_client.get_adapter('https://').max_retries + retry = retry.increment(error=error) + retry = retry.increment(error=error) + retry = retry.increment(error=error) + retry = retry.increment(error=error) + with pytest.raises(MaxRetryError) as retry_err: + retry.increment(error=error) + assert retry_err.value.reason == error + +def test_retry_config_disable(): + # Test disabling retries + service = BaseService(service_url='https://mockurl/', authenticator=NoAuthAuthenticator()) + service.enable_retries() + service.disable_retries() + assert service.retry_config is None + assert service.http_client.get_adapter('https://').max_retries.total == 0 + + # Ensure retries are not started after one connection attempt + error = ConnectTimeoutError() + retry = service.http_client.get_adapter('https://').max_retries + with pytest.raises(MaxRetryError) as retry_err: + retry.increment(error=error) + assert retry_err.value.reason == error + +def test_retry_config_non_default(): + service = BaseService(service_url='https://mockurl/', authenticator=NoAuthAuthenticator()) + service.enable_retries(2, 0.3) + assert service.retry_config.total == 2 + assert service.retry_config.backoff_factor == 0.3 + + # Ensure retries fail after 2 retries + error = ConnectTimeoutError() + retry = service.http_client.get_adapter('https://').max_retries + retry = retry.increment(error=error) + retry = retry.increment(error=error) + with pytest.raises(MaxRetryError) as retry_err: + retry.increment(error=error) + assert retry_err.value.reason == error + +def test_retry_config_external(): + file_path = os.path.join( + os.path.dirname(__file__), '../resources/ibm-credentials-retry.env') + os.environ['IBM_CREDENTIALS_FILE'] = file_path + service = IncludeExternalConfigService('v1', authenticator=NoAuthAuthenticator()) + assert service.retry_config.total == 3 + assert service.retry_config.backoff_factor == 0.2 + + # Ensure retries fail after 3 retries + error = ConnectTimeoutError() + retry = service.http_client.get_adapter('https://').max_retries + retry = retry.increment(error=error) + retry = retry.increment(error=error) + retry = retry.increment(error=error) + with pytest.raises(MaxRetryError) as retry_err: + retry.increment(error=error) + assert retry_err.value.reason == error + @responses.activate def test_user_agent_header(): service = AnyServiceV1('2018-11-20', authenticator=NoAuthAuthenticator())