feat(python): add max retry configuration for python requests session

.gitignore

@@ -53,6 +53,7 @@ venv*/
 python3/
 
 *.env
+!resources/*.env
 
 .sfdx/tools/apex.db
 .pytest_cache/

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "package-lock.json|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2021-02-24T18:40:26Z",
+  "generated_at": "2021-03-18T14:18:04Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -132,6 +132,16 @@
         "verified_result": null
       }
     ],
+    "resources/ibm-credentials-retry.env": [
+      {
+        "hashed_secret": "ce49dd46e23153d6593eccd68534b9f1465d5bbd",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 1,
+        "type": "Secret Keyword",
+        "verified_result": null
+      }
+    ],
     "resources/ibm-credentials.env": [
       {
         "hashed_secret": "b9cad336062c0dc3bb30145b1a6697fccfe755a6",
@@ -219,23 +229,23 @@
         "hashed_secret": "da2f27d2c57a0e1ed2dc3a34b4ef02faf2f7a4c2",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 27,
+        "line_number": 29,
         "type": "Hex High Entropy String",
         "verified_result": null
       },
       {
         "hashed_secret": "b3f00e146afe19aab0069029b7fb3926ad756d26",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 99,
+        "line_number": 107,
         "type": "Hex High Entropy String",
         "verified_result": null
       },
       {
         "hashed_secret": "62cdb7020ff920e5aa642c3d4066950dd1f01f4d",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 159,
+        "line_number": 170,
         "type": "Secret Keyword",
         "verified_result": null
       }
@@ -255,23 +265,23 @@
         "hashed_secret": "34a0a47a51d5bf739df0214450385e29ee7e9847",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 253,
+        "line_number": 353,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "2863fa4b5510c46afc2bd2998dfbc0cf3d6df032",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 329,
+        "line_number": 429,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "b9cad336062c0dc3bb30145b1a6697fccfe755a6",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 390,
+        "line_number": 490,
         "type": "Secret Keyword",
         "verified_result": null
       }

ibm_cloud_sdk_core/base_service.py

@@ -24,7 +24,9 @@
 from typing import Dict, List, Optional, Tuple, Union
 
 import requests
+from requests.adapters import HTTPAdapter
 from requests.structures import CaseInsensitiveDict
+from urllib3.util.retry import Retry
 from ibm_cloud_sdk_core.authenticators import Authenticator
 from .version import __version__
 from .utils import (has_bad_first_or_last_char, remove_null_values,
@@ -88,11 +90,35 @@ def __init__(self,
         self.default_headers = None
         self.enable_gzip_compression = enable_gzip_compression
         self._set_user_agent_header(self._build_user_agent())
+        self.retry_config = None
+        self.http_adapter = HTTPAdapter()
         if not self.authenticator:
             raise ValueError('authenticator must be provided')
         if not isinstance(self.authenticator, Authenticator):
             raise ValueError('authenticator should be of type Authenticator')
 
+    def enable_retries(self, max_retries: int = 4, retry_interval: float = 0.1) -> None:
+        """Setup http_client with retry_config and http_adapter"""
+        self.retry_config = Retry(
+            total = max_retries,
+            backoff_factor = retry_interval,
+            # List of HTTP status codes to retry on in addition to Timeout/Connection Errors
+            status_forcelist = [429, 500, 502, 503, 504],
+            # List of HTTP methods to retry on
+            # Omitting this will default to all methods except POST
+            allowed_methods=['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST']
+        )
+        self.http_adapter = HTTPAdapter(max_retries=self.retry_config)
+        self.http_client.mount('http://', self.http_adapter)
+        self.http_client.mount('https://', self.http_adapter)
+
+    def disable_retries(self):
+        """Remove retry config from http_adapter"""
+        self.retry_config = None
+        self.http_adapter = HTTPAdapter()
+        self.http_client.mount('http://', self.http_adapter)
+        self.http_client.mount('https://', self.http_adapter)
+
     @staticmethod
     def _get_system_info() -> str:
         return '{0} {1} {2}'.format(
@@ -126,10 +152,19 @@ def configure_service(self, service_name: str) -> None:
         if config.get('URL'):
             self.set_service_url(config.get('URL'))
         if config.get('DISABLE_SSL'):
-            self.set_disable_ssl_verification(bool(config.get('DISABLE_SSL')))
-        if config.get('ENABLE_GZIP') is not None:
+            self.set_disable_ssl_verification(
+                config.get('DISABLE_SSL').lower() == 'true')
+        if config.get('ENABLE_GZIP'):
             self.set_enable_gzip_compression(
-                config.get('ENABLE_GZIP') == 'True')
+                config.get('ENABLE_GZIP').lower() == 'true')
+        if config.get('ENABLE_RETRIES'):
+            if config.get('ENABLE_RETRIES').lower() == 'true':
+                kwargs = {}
+                if config.get('MAX_RETRIES'):
+                    kwargs["max_retries"] = int(config.get('MAX_RETRIES'))
+                if config.get('RETRY_INTERVAL'):
+                    kwargs["retry_interval"] = float(config.get('RETRY_INTERVAL'))
+                self.enable_retries(**kwargs)
 
     def _set_user_agent_header(self, user_agent_string: str) -> None:
         self.user_agent_header = {'User-Agent': user_agent_string}

resources/ibm-credentials-retry.env

@@ -0,0 +1,6 @@
+INCLUDE_EXTERNAL_CONFIG_APIKEY=mockkey
+INCLUDE_EXTERNAL_CONFIG_AUTH_TYPE=iam
+INCLUDE_EXTERNAL_CONFIG_URL=https://mockurl
+INCLUDE_EXTERNAL_CONFIG_MAX_RETRIES=3
+INCLUDE_EXTERNAL_CONFIG_RETRY_INTERVAL=0.2
+INCLUDE_EXTERNAL_CONFIG_ENABLE_RETRIES=true

test/test_base_service.py

@@ -11,6 +11,7 @@
 import responses
 import requests
 import jwt
+from urllib3.exceptions import ConnectTimeoutError, MaxRetryError
 from ibm_cloud_sdk_core import BaseService, DetailedResponse
 from ibm_cloud_sdk_core import ApiException
 from ibm_cloud_sdk_core import CP4DTokenManager
@@ -553,6 +554,72 @@ def test_gzip_compression_external():
     assert prepped['data'] == gzip.compress(b'{"foo": "bar"}')
     assert prepped['headers'].get('content-encoding') == 'gzip'
 
+def test_retry_config_default():
+    service = BaseService(service_url='https://mockurl/', authenticator=NoAuthAuthenticator())
+    service.enable_retries()
+    assert service.retry_config.total == 4
+    assert service.retry_config.backoff_factor == 0.1
+    assert service.http_client.get_adapter('https://').max_retries.total == 4
+
+    # Ensure retries fail after 4 retries
+    error = ConnectTimeoutError()
+    retry = service.http_client.get_adapter('https://').max_retries
+    retry = retry.increment(error=error)
+    retry = retry.increment(error=error)
+    retry = retry.increment(error=error)
+    retry = retry.increment(error=error)
+    with pytest.raises(MaxRetryError) as retry_err:
+        retry.increment(error=error)
+    assert retry_err.value.reason == error
+
+def test_retry_config_disable():
+    # Test disabling retries
+    service = BaseService(service_url='https://mockurl/', authenticator=NoAuthAuthenticator())
+    service.enable_retries()
+    service.disable_retries()
+    assert service.retry_config is None
+    assert service.http_client.get_adapter('https://').max_retries.total == 0
+
+    # Ensure retries are not started after one connection attempt
+    error = ConnectTimeoutError()
+    retry = service.http_client.get_adapter('https://').max_retries
+    with pytest.raises(MaxRetryError) as retry_err:
+        retry.increment(error=error)
+    assert retry_err.value.reason == error
+
+def test_retry_config_non_default():
+    service = BaseService(service_url='https://mockurl/', authenticator=NoAuthAuthenticator())
+    service.enable_retries(2, 0.3)
+    assert service.retry_config.total == 2
+    assert service.retry_config.backoff_factor == 0.3
+
+    # Ensure retries fail after 2 retries
+    error = ConnectTimeoutError()
+    retry = service.http_client.get_adapter('https://').max_retries
+    retry = retry.increment(error=error)
+    retry = retry.increment(error=error)
+    with pytest.raises(MaxRetryError) as retry_err:
+        retry.increment(error=error)
+    assert retry_err.value.reason == error
+
+def test_retry_config_external():
+    file_path = os.path.join(
+        os.path.dirname(__file__), '../resources/ibm-credentials-retry.env')
+    os.environ['IBM_CREDENTIALS_FILE'] = file_path
+    service = IncludeExternalConfigService('v1', authenticator=NoAuthAuthenticator())
+    assert service.retry_config.total == 3
+    assert service.retry_config.backoff_factor == 0.2
+
+    # Ensure retries fail after 3 retries
+    error = ConnectTimeoutError()
+    retry = service.http_client.get_adapter('https://').max_retries
+    retry = retry.increment(error=error)
+    retry = retry.increment(error=error)
+    retry = retry.increment(error=error)
+    with pytest.raises(MaxRetryError) as retry_err:
+        retry.increment(error=error)
+    assert retry_err.value.reason == error
+
 @responses.activate
 def test_user_agent_header():
     service = AnyServiceV1('2018-11-20', authenticator=NoAuthAuthenticator())