Merge branch 'master' into master

eng/pipelines/templates/steps/analyze.yml

@@ -91,8 +91,8 @@ steps:
       BuildTargetingString: ${{ parameters.BuildTargetingString }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
 
-  - template: ../steps/run_apistub.yml
-    parameters: 
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      BuildTargetingString: ${{ parameters.BuildTargetingString }}
-      TestMarkArgument: ${{ parameters.TestMarkArgument }}
+  # - template: ../steps/run_apistub.yml
+  #   parameters:
+  #     ServiceDirectory: ${{ parameters.ServiceDirectory }}
+  #     BuildTargetingString: ${{ parameters.BuildTargetingString }}
+  #     TestMarkArgument: ${{ parameters.TestMarkArgument }}

eng/tox/install_dev_build_dependency.py

@@ -13,13 +13,11 @@
 from os import path
 from subprocess import check_call
 
-from pip._internal.operations import freeze
-
 # import common_task module
 root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", ".."))
 common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks"))
 sys.path.append(common_task_path)
-from common_tasks import process_glob_string
+from common_tasks import process_glob_string, get_installed_packages
 from tox_helper_tasks import get_package_details
 
 EXCLUDED_PKGS = [
@@ -34,10 +32,10 @@
 # This script verifies installed package version and ensure all installed pacakges are dev build version
 
 
-def get_installed_packages(pkg_name_to_exclude):
+def get_installed_azure_packages(pkg_name_to_exclude):
     # This method returns a list of installed azure sdk packages
     installed_pkgs = [
-        p.split("==")[0] for p in freeze.freeze() if p.startswith("azure-")
+        p.split("==")[0] for p in get_installed_packages() if p.startswith("azure-")
     ]
 
     # Get valid list of Azure SDK packages in repo
@@ -100,7 +98,7 @@ def install_packages(packages):
 
 def install_dev_build_packages(pkg_name_to_exclude):
     # Uninstall GA version and reinstall dev build version of dependent packages
-    azure_pkgs = get_installed_packages(pkg_name_to_exclude)
+    azure_pkgs = get_installed_azure_packages(pkg_name_to_exclude)
     uninstall_packages(azure_pkgs)
     install_packages(azure_pkgs)
 

eng/tox/verify_installed_packages.py

@@ -9,8 +9,13 @@
 import os
 import sys
 import logging
-from pip._internal.operations import freeze
+from os import path
 
+# import common_task module
+root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", ".."))
+common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks"))
+sys.path.append(common_task_path)
+from common_tasks import get_installed_packages
 
 def verify_packages(package_file_path):
     # this method verifies packages installed on machine is matching the expected package version
@@ -29,7 +34,7 @@ def verify_packages(package_file_path):
         sys.exit(1)
 
     # find installed and expected packages
-    installed = dict(p.split('==') for p in freeze.freeze() if p.startswith('azure') and "==" in p)
+    installed = dict(p.split('==') for p in get_installed_packages() if p.startswith('azure') and "==" in p)
     expected = dict(p.split('==') for p in packages)
 
     missing_packages = [pkg for pkg in expected.keys() if installed.get(pkg) != expected.get(pkg)]

scripts/devops_tasks/common_tasks.py

@@ -22,7 +22,7 @@
 import pdb
 
 # Assumes the presence of setuptools
-from pkg_resources import parse_version, parse_requirements, Requirement
+from pkg_resources import parse_version, parse_requirements, Requirement, WorkingSet, working_set
 
 # this assumes the presence of "packaging"
 from packaging.specifiers import SpecifierSet
@@ -420,4 +420,15 @@ def find_tools_packages(root_path):
     glob_string = os.path.join(root_path, "tools", "*", "setup.py")
     pkgs = [os.path.basename(os.path.dirname(p)) for p in glob.glob(glob_string)]
     logging.info("Packages in tools: {}".format(pkgs))
-    return pkgs
+    return pkgs
+
+
+def get_installed_packages(paths = None):
+    """Find packages in default or given lib paths
+    """
+    # WorkingSet returns installed packages in given path
+    # working_set returns installed packages in default path
+    # if paths is set then find installed packages from given paths
+    ws = WorkingSet(paths) if paths else working_set  
+    return ["{0}=={1}".format(p.project_name, p.version) for p in ws]
+

scripts/devops_tasks/test_regression.py

@@ -22,10 +22,10 @@
     filter_dev_requirements,
     find_packages_missing_on_pypi,
     find_whl,
-    find_tools_packages
+    find_tools_packages,
+    get_installed_packages
 )
 from git_helper import get_release_tag, git_checkout_tag, git_checkout_branch, clone_repo
-from pip._internal.operations import freeze
 
 AZURE_GLOB_STRING = "azure*"
 
@@ -222,7 +222,7 @@ def _install_packages(self, dependent_pkg_path, pkg_to_exclude):
         temp_dir = self.context.temp_path
 
         list_to_exclude = [pkg_to_exclude,]
-        installed_pkgs = [p.split('==')[0] for p in list(freeze.freeze(paths=self.context.venv.lib_paths)) if p.startswith('azure-')]
+        installed_pkgs = [p.split('==')[0] for p in get_installed_packages(self.context.venv.lib_paths) if p.startswith('azure-')]
         logging.info("Installed azure sdk packages:{}".format(installed_pkgs))
 
         # Do not exclude list of packages in tools directory and so these tools packages will be reinstalled from repo branch we are testing
@@ -257,7 +257,7 @@ def _is_package_installed(self, package, version):
         venv_root = self.context.venv.path
         site_packages = self.context.venv.lib_paths
         logging.info("Searching for packages in :{}".format(site_packages))
-        installed_pkgs = list(freeze.freeze(paths=site_packages))
+        installed_pkgs = get_installed_packages(site_packages)
         logging.info("Installed packages: {}".format(installed_pkgs))
         # Verify installed package version
         # Search for exact version or dev build version of current version.

sdk/cosmos/azure-cosmos/CHANGELOG.md

@@ -1,13 +1,16 @@
 ## 4.0.1 (Unreleased)
 
 - Added deprecation warning for "lazy" indexing mode. The backend no longer allows creating containers with this mode and will set them to consistent instead.
-- Fix for bug where options headers were not added to upsert_item function. Issue #11791 - thank you @aalapatirvbd.
-- Fixed error raised when a non string ID is used in an item. It now raises TypeError rather than AttributeError. Issue #11793 - thank you @Rabbit994.
+
+**New features**
 - Added the ability to set the analytical storage TTL when creating a new container.
 
-** Bug fixes **
+**Bug fixes**
 - Fixed support for dicts as inputs for get_client APIs.
 - Fixed Python 2/3 compatibility in query iterators.
+- Fixed type hint error. Issue #12570 - Thanks @sl-sandy.
+- Fixed bug where options headers were not added to upsert_item function. Issue #11791 - thank you @aalapatirvbd.
+- Fixed error raised when a non string ID is used in an item. It now raises TypeError rather than AttributeError. Issue #11793 - thank you @Rabbit994.
 
 
 ## 4.0.0 (2020-05-20)

sdk/cosmos/azure-cosmos/azure/cosmos/container.py

@@ -282,7 +282,7 @@ def query_items_change_feed(
     def query_items(
         self,
         query,  # type: str
-        parameters=None,  # type: Optional[List[str]]
+        parameters=None,  # type: Optional[List[Dict[str, object]]]
         partition_key=None,  # type: Optional[Any]
         enable_cross_partition_query=None,  # type: Optional[bool]
         max_item_count=None,  # type: Optional[int]
@@ -299,7 +299,9 @@ def query_items(
         the WHERE clause.
 
         :param query: The Azure Cosmos DB SQL query to execute.
-        :param parameters: Optional array of parameters to the query. Ignored if no query is provided.
+        :param parameters: Optional array of parameters to the query.
+            Each parameter is a dict() with 'name' and 'value' keys.
+            Ignored if no query is provided.
         :param partition_key: Specifies the partition key value for the item.
         :param enable_cross_partition_query: Allows sending of more than one request to
             execute the query in the Azure Cosmos DB service.

sdk/formrecognizer/azure-ai-formrecognizer/README.md

@@ -193,7 +193,7 @@ all the recognize methods.
 
 
 ```
-form_url_jpg = "<url_of_the_form>"
+form_url = "<url_of_the_form>"
 poller = form_recognizer_client.begin_recognize_custom_forms_from_url(model_id=model_id, form_url=form_url)
 result = poller.result()
 ```

sdk/formrecognizer/azure-ai-formrecognizer/azure/ai/formrecognizer/_form_recognizer_client.py

@@ -94,7 +94,7 @@ def begin_recognize_receipts(self, receipt, **kwargs):
         'image/jpeg', 'image/png' or 'image/tiff'.
 
         See fields found on a receipt here:
-        https://aka.ms/azsdk/python/formrecognizer/receiptfields
+        https://aka.ms/formrecognizer/receiptfields
 
         :param receipt: JPEG, PNG, PDF and TIFF type file stream or bytes.
              Currently only supports US sales receipts.
@@ -152,7 +152,7 @@ def begin_recognize_receipts_from_url(self, receipt_url, **kwargs):
         The input document must be the location (URL) of the receipt to be analyzed.
 
         See fields found on a receipt here:
-        https://aka.ms/azsdk/python/formrecognizer/receiptfields
+        https://aka.ms/formrecognizer/receiptfields
 
         :param str receipt_url: The URL of the receipt to analyze. The input must be a valid, encoded URL
             of one of the supported formats: JPEG, PNG, PDF and TIFF. Currently only supports

sdk/formrecognizer/azure-ai-formrecognizer/azure/ai/formrecognizer/aio/_form_recognizer_client_async.py

@@ -102,7 +102,7 @@ async def begin_recognize_receipts(
         'image/jpeg', 'image/png' or 'image/tiff'.
 
         See fields found on a receipt here:
-        https://aka.ms/azsdk/python/formrecognizer/receiptfields
+        https://aka.ms/formrecognizer/receiptfields
 
         :param receipt: JPEG, PNG, PDF and TIFF type file stream or bytes.
             Currently only supports US sales receipts.
@@ -166,7 +166,7 @@ async def begin_recognize_receipts_from_url(
         The input document must be the location (URL) of the receipt to be analyzed.
 
         See fields found on a receipt here:
-        https://aka.ms/azsdk/python/formrecognizer/receiptfields
+        https://aka.ms/formrecognizer/receiptfields
 
         :param str receipt_url: The URL of the receipt to analyze. The input must be a valid, encoded URL
             of one of the supported formats: JPEG, PNG, PDF and TIFF. Currently only supports

sdk/formrecognizer/azure-ai-formrecognizer/samples/async_samples/sample_recognize_receipts_async.py

@@ -15,7 +15,7 @@
     from receipts, see sample_strongly_typed_recognized_form_async.py.
 
     See fields found on a receipt here:
-    https://aka.ms/azsdk/python/formrecognizer/receiptfields
+    https://aka.ms/formrecognizer/receiptfields
 
 USAGE:
     python sample_recognize_receipts_async.py

sdk/formrecognizer/azure-ai-formrecognizer/samples/async_samples/sample_recognize_receipts_from_url_async.py

@@ -15,7 +15,7 @@
     from receipts, see sample_strongly_typed_recognized_form_async.py.
 
     See fields found on a receipt here:
-    https://aka.ms/azsdk/python/formrecognizer/receiptfields
+    https://aka.ms/formrecognizer/receiptfields
 
 USAGE:
     python sample_recognize_receipts_from_url_async.py

sdk/formrecognizer/azure-ai-formrecognizer/samples/async_samples/sample_strongly_typing_recognized_form_async.py

@@ -16,7 +16,7 @@
     fields' names and types.
 
     See fields found on a receipt here:
-    https://aka.ms/azsdk/python/formrecognizer/receiptfields
+    https://aka.ms/formrecognizer/receiptfields
 
 USAGE:
     python sample_strongly_typed_recognized_form_async.py
@@ -36,7 +36,7 @@ class Receipt(object):
     If a specific field is not found on the receipt, it will return None.
 
     See fields found on a receipt here:
-    https://aka.ms/azsdk/python/formrecognizer/receiptfields
+    https://aka.ms/formrecognizer/receiptfields
     """
 
     def __init__(self, form):

sdk/formrecognizer/azure-ai-formrecognizer/samples/sample_recognize_receipts.py

@@ -15,7 +15,7 @@
     from receipts, see sample_strongly_typed_recognized_form.py.
 
     See fields found on a receipt here:
-    https://aka.ms/azsdk/python/formrecognizer/receiptfields
+    https://aka.ms/formrecognizer/receiptfields
 
 USAGE:
     python sample_recognize_receipts.py

sdk/formrecognizer/azure-ai-formrecognizer/samples/sample_recognize_receipts_from_url.py

@@ -15,7 +15,7 @@
     from receipts, see sample_strongly_typed_recognized_form.py.
 
     See fields found on a receipt here:
-    https://aka.ms/azsdk/python/formrecognizer/receiptfields
+    https://aka.ms/formrecognizer/receiptfields
 
 USAGE:
     python sample_recognize_receipts_from_url.py

sdk/formrecognizer/azure-ai-formrecognizer/samples/sample_strongly_typing_recognized_form.py

@@ -16,7 +16,7 @@
     fields' names and types.
 
     See fields found on a receipt here:
-    https://aka.ms/azsdk/python/formrecognizer/receiptfields
+    https://aka.ms/formrecognizer/receiptfields
 
 USAGE:
     python sample_strongly_typed_recognized_form.py
@@ -35,7 +35,7 @@ class Receipt(object):
     If a specific field is not found on the receipt, it will return None.
 
     See fields found on a receipt here:
-    https://aka.ms/azsdk/python/formrecognizer/receiptfields
+    https://aka.ms/formrecognizer/receiptfields
     """
 
     def __init__(self, form):

sdk/identity/azure-identity/MANIFEST.in

@@ -1,3 +1,4 @@
+recursive-include samples *.py
 recursive-include tests *.py
 include *.md
-include azure/__init__.py
+include azure/__init__.py

sdk/identity/azure-identity/azure/identity/_authn_client.py

@@ -22,7 +22,7 @@
     UserAgentPolicy,
 )
 from azure.core.pipeline.transport import RequestsTransport, HttpRequest
-from ._constants import AZURE_CLI_CLIENT_ID
+from ._constants import AZURE_CLI_CLIENT_ID, DEFAULT_REFRESH_OFFSET, DEFAULT_TOKEN_REFRESH_RETRY_DELAY
 from ._internal import get_default_authority, normalize_authority
 from ._internal.user_agent import USER_AGENT
 
@@ -65,17 +65,32 @@ def __init__(self, endpoint=None, authority=None, tenant=None, **kwargs):  # pyl
             authority = normalize_authority(authority) if authority else get_default_authority()
             self._auth_url = "/".join((authority, tenant.strip("/"), "oauth2/v2.0/token"))
         self._cache = kwargs.get("cache") or TokenCache()  # type: TokenCache
+        self._token_refresh_retry_delay = DEFAULT_TOKEN_REFRESH_RETRY_DELAY
+        self._token_refresh_offset = DEFAULT_REFRESH_OFFSET
+        self._last_refresh_time = 0
 
     @property
     def auth_url(self):
         return self._auth_url
 
+    def should_refresh(self, token):
+        # type: (AccessToken) -> bool
+        """ check if the token needs refresh or not
+        """
+        expires_on = int(token.expires_on)
+        now = int(time.time())
+        if expires_on - now > self._token_refresh_offset:
+            return False
+        if now - self._last_refresh_time < self._token_refresh_retry_delay:
+            return False
+        return True
+
     def get_cached_token(self, scopes):
         # type: (Iterable[str]) -> Optional[AccessToken]
         tokens = self._cache.find(TokenCache.CredentialType.ACCESS_TOKEN, target=list(scopes))
         for token in tokens:
             expires_on = int(token["expires_on"])
-            if expires_on - 300 > int(time.time()):
+            if expires_on > int(time.time()):
                 return AccessToken(token["secret"], expires_on)
         return None
 
@@ -217,6 +232,7 @@ def request_token(
         # type: (...) -> AccessToken
         request = self._prepare_request(method, headers=headers, form_data=form_data, params=params)
         request_time = int(time.time())
+        self._last_refresh_time = request_time   # no matter succeed or not, update the last refresh time
         response = self._pipeline.run(request, stream=False, **kwargs)
         token = self._deserialize_and_cache_token(response=response, scopes=scopes, request_time=request_time)
         return token

sdk/identity/azure-identity/azure/identity/_constants.py

@@ -7,6 +7,8 @@
 AZURE_CLI_CLIENT_ID = "04b07795-8ddb-461a-bbee-02f9e1bf7b46"
 AZURE_VSCODE_CLIENT_ID = "aebc6443-996d-45c2-90f0-388ff96faa56"
 VSCODE_CREDENTIALS_SECTION = "VS Code Azure"
+DEFAULT_REFRESH_OFFSET = 300
+DEFAULT_TOKEN_REFRESH_RETRY_DELAY = 30
 
 
 class KnownAuthorities: