Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Datalake] Added support for PurePosixPath #16400

Merged
merged 6 commits into from
Feb 1, 2021
Merged

[Datalake] Added support for PurePosixPath #16400

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9d2a3eb
Added support for PurePosixPath
tasherif-msft Jan 28, 2021
08a55a6
removed posix lib from imports
tasherif-msft Jan 28, 2021
0fbce3e
removed posix lib from imports
tasherif-msft Jan 28, 2021
a646c45
added type annotations
tasherif-msft Jan 28, 2021
7b22145
cleaned up changes
tasherif-msft Feb 1, 2021
f54ec27
fixed merge conflict
tasherif-msft Feb 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 19 additions & 13 deletions ...age/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from ._deserialize import deserialize_dir_properties
from ._shared.base_client import TransportWrapper, parse_connection_str
from ._data_lake_file_client import DataLakeFileClient
from ._models import DirectoryProperties
from ._models import DirectoryProperties, FileProperties
from ._path_client import PathClient


Expand Down Expand Up @@ -490,7 +490,7 @@ def create_file(self, file, # type: Union[FileProperties, str]
file_client.create_file(**kwargs)
return file_client

def get_file_client(self, file # type: Union[FileProperties, str]
def get_file_client(self, file # type: Union[FileProperties, str, PurePosixPath]
):
# type: (...) -> DataLakeFileClient
"""Get a client to interact with the specified file.
Expand All @@ -500,14 +500,17 @@ def get_file_client(self, file # type: Union[FileProperties, str]
:param file:
The file with which to interact. This can either be the name of the file,
or an instance of FileProperties. eg. directory/subdirectory/file
:type file: str or ~azure.storage.filedatalake.FileProperties
:type file: str or ~azure.storage.filedatalake.FileProperties or a PurePosixPath
:returns: A DataLakeFileClient.
:rtype: ~azure.storage.filedatalake..DataLakeFileClient
"""
try:
file_path = file.name
except AttributeError:
file_path = self.path_name + '/' + file
tasherif-msft marked this conversation as resolved.
Show resolved Hide resolved
if not isinstance(file, FileProperties) and hasattr(file, "name"):
file_path = str(file)
else:
try:
file_path = file.name
except AttributeError:
file_path = self.path_name + '/' + file

_pipeline = Pipeline(
transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access
Expand All @@ -520,7 +523,7 @@ def get_file_client(self, file # type: Union[FileProperties, str]
key_encryption_key=self.key_encryption_key,
key_resolver_function=self.key_resolver_function)

def get_sub_directory_client(self, sub_directory # type: Union[DirectoryProperties, str]
def get_sub_directory_client(self, sub_directory # type: Union[DirectoryProperties, str, PurePosixPath]
):
# type: (...) -> DataLakeDirectoryClient
"""Get a client to interact with the specified subdirectory of the current directory.
Expand All @@ -530,14 +533,17 @@ def get_sub_directory_client(self, sub_directory # type: Union[DirectoryPropert
:param sub_directory:
The directory with which to interact. This can either be the name of the directory,
or an instance of DirectoryProperties.
:type sub_directory: str or ~azure.storage.filedatalake.DirectoryProperties
:type sub_directory: str or ~azure.storage.filedatalake.DirectoryProperties or a PurePosixPath
:returns: A DataLakeDirectoryClient.
:rtype: ~azure.storage.filedatalake.DataLakeDirectoryClient
"""
try:
subdir_path = sub_directory.name
except AttributeError:
subdir_path = self.path_name + '/' + sub_directory
if not isinstance(sub_directory, DirectoryProperties) and hasattr(sub_directory, "name"):
subdir_path = str(sub_directory)
else:
try:
subdir_path = sub_directory.name
except AttributeError:
subdir_path = self.path_name + '/' + sub_directory

_pipeline = Pipeline(
transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access
Expand Down
32 changes: 19 additions & 13 deletions sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from azure.storage.blob import ContainerClient
from ._shared.base_client import TransportWrapper, StorageAccountHostsMixin, parse_query, parse_connection_str
from ._serialize import convert_dfs_url_to_blob_url
from ._models import LocationMode, FileSystemProperties, PublicAccess
from ._models import LocationMode, FileSystemProperties, PublicAccess, FileProperties, DirectoryProperties
from ._list_paths_helper import PathPropertiesPaged
from ._data_lake_file_client import DataLakeFileClient
from ._data_lake_directory_client import DataLakeDirectoryClient
Expand Down Expand Up @@ -714,7 +714,7 @@ def _get_root_directory_client(self):
"""
return self.get_directory_client('/')

def get_directory_client(self, directory # type: Union[DirectoryProperties, str]
def get_directory_client(self, directory # type: Union[DirectoryProperties, str, PurePosixPath]
):
# type: (...) -> DataLakeDirectoryClient
"""Get a client to interact with the specified directory.
Expand All @@ -723,7 +723,7 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str

:param directory:
The directory with which to interact. This can either be the name of the directory,
or an instance of DirectoryProperties.
or an instance of DirectoryProperties or a PurePosixPath.
:type directory: str or ~azure.storage.filedatalake.DirectoryProperties
:returns: A DataLakeDirectoryClient.
:rtype: ~azure.storage.filedatalake.DataLakeDirectoryClient
Expand All @@ -737,10 +737,13 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str
:dedent: 8
:caption: Getting the directory client to interact with a specific directory.
"""
try:
directory_name = directory.name
except AttributeError:
directory_name = directory
if not isinstance(directory, DirectoryProperties) and hasattr(directory, "name"):
directory_name = str(directory)
else:
try:
directory_name = directory.name
except AttributeError:
directory_name = directory
_pipeline = Pipeline(
transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access
policies=self._pipeline._impl_policies # pylint: disable = protected-access
Expand All @@ -754,7 +757,7 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str
key_resolver_function=self.key_resolver_function
)

def get_file_client(self, file_path # type: Union[FileProperties, str]
def get_file_client(self, file_path # type: Union[FileProperties, str, PurePosixPath]
):
# type: (...) -> DataLakeFileClient
"""Get a client to interact with the specified file.
Expand All @@ -764,7 +767,7 @@ def get_file_client(self, file_path # type: Union[FileProperties, str]
:param file_path:
The file with which to interact. This can either be the path of the file(from root directory),
or an instance of FileProperties. eg. directory/subdirectory/file
:type file_path: str or ~azure.storage.filedatalake.FileProperties
:type file_path: str or ~azure.storage.filedatalake.FileProperties or a PurePosixPath
:returns: A DataLakeFileClient.
:rtype: ~azure.storage.filedatalake..DataLakeFileClient

Expand All @@ -777,10 +780,13 @@ def get_file_client(self, file_path # type: Union[FileProperties, str]
:dedent: 8
:caption: Getting the file client to interact with a specific file.
"""
try:
file_path = file_path.name
except AttributeError:
pass
if not isinstance(file_path, FileProperties) and hasattr(file_path, "name"):
file_path = str(file_path)
else:
try:
file_path = file_path.name
except AttributeError:
pass
_pipeline = Pipeline(
transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access
policies=self._pipeline._impl_policies # pylint: disable = protected-access
Expand Down
32 changes: 19 additions & 13 deletions ...storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from azure.core.pipeline import AsyncPipeline
from ._data_lake_file_client_async import DataLakeFileClient
from .._data_lake_directory_client import DataLakeDirectoryClient as DataLakeDirectoryClientBase
from .._models import DirectoryProperties
from .._models import DirectoryProperties, FileProperties
from .._deserialize import deserialize_dir_properties
from ._path_client_async import PathClient
from .._shared.base_client_async import AsyncTransportWrapper
Expand Down Expand Up @@ -459,7 +459,7 @@ async def create_file(self, file, # type: Union[FileProperties, str]
await file_client.create_file(**kwargs)
return file_client

def get_file_client(self, file # type: Union[FileProperties, str]
def get_file_client(self, file # type: Union[FileProperties, str, PurePosixPath]
):
# type: (...) -> DataLakeFileClient
"""Get a client to interact with the specified file.
Expand All @@ -469,7 +469,7 @@ def get_file_client(self, file # type: Union[FileProperties, str]
:param file:
The file with which to interact. This can either be the name of the file,
or an instance of FileProperties. eg. directory/subdirectory/file
:type file: str or ~azure.storage.filedatalake.FileProperties
:type file: str or ~azure.storage.filedatalake.FileProperties or PurePosixPath
:returns: A DataLakeFileClient.
:rtype: ~azure.storage.filedatalake.aio.DataLakeFileClient

Expand All @@ -482,10 +482,13 @@ def get_file_client(self, file # type: Union[FileProperties, str]
:dedent: 12
:caption: Getting the file client to interact with a specific file.
"""
try:
file_path = file.name
except AttributeError:
file_path = self.path_name + '/' + file
if not isinstance(file, FileProperties) and hasattr(file, "name"):
file_path = str(file)
else:
try:
file_path = file.name
except AttributeError:
file_path = self.path_name + '/' + file

_pipeline = AsyncPipeline(
transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable = protected-access
Expand All @@ -498,7 +501,7 @@ def get_file_client(self, file # type: Union[FileProperties, str]
key_encryption_key=self.key_encryption_key,
key_resolver_function=self.key_resolver_function)

def get_sub_directory_client(self, sub_directory # type: Union[DirectoryProperties, str]
def get_sub_directory_client(self, sub_directory # type: Union[DirectoryProperties, str, PurePosixPath]
):
# type: (...) -> DataLakeDirectoryClient
"""Get a client to interact with the specified subdirectory of the current directory.
Expand All @@ -508,7 +511,7 @@ def get_sub_directory_client(self, sub_directory # type: Union[DirectoryPropert
:param sub_directory:
The directory with which to interact. This can either be the name of the directory,
or an instance of DirectoryProperties.
:type sub_directory: str or ~azure.storage.filedatalake.DirectoryProperties
:type sub_directory: str or ~azure.storage.filedatalake.DirectoryProperties or a PurePosixPath
:returns: A DataLakeDirectoryClient.
:rtype: ~azure.storage.filedatalake.aio.DataLakeDirectoryClient

Expand All @@ -521,10 +524,13 @@ def get_sub_directory_client(self, sub_directory # type: Union[DirectoryPropert
:dedent: 12
:caption: Getting the directory client to interact with a specific directory.
"""
try:
subdir_path = sub_directory.name
except AttributeError:
subdir_path = self.path_name + '/' + sub_directory
if not isinstance(sub_directory, DirectoryProperties) and hasattr(sub_directory, "name"):
subdir_path = str(sub_directory)
else:
try:
subdir_path = sub_directory.name
except AttributeError:
subdir_path = self.path_name + '/' + sub_directory

_pipeline = AsyncPipeline(
transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable = protected-access
Expand Down
32 changes: 19 additions & 13 deletions ...e/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from .._generated.aio import AzureDataLakeStorageRESTAPI
from .._shared.base_client_async import AsyncTransportWrapper, AsyncStorageAccountHostsMixin
from .._shared.policies_async import ExponentialRetry
from .._models import FileSystemProperties, PublicAccess
from .._models import FileSystemProperties, PublicAccess, DirectoryProperties, FileProperties

if TYPE_CHECKING:
from datetime import datetime
Expand Down Expand Up @@ -674,7 +674,7 @@ def _get_root_directory_client(self):
"""
return self.get_directory_client('/')

def get_directory_client(self, directory # type: Union[DirectoryProperties, str]
def get_directory_client(self, directory # type: Union[DirectoryProperties, str, PurePosixPath]
):
# type: (...) -> DataLakeDirectoryClient
"""Get a client to interact with the specified directory.
Expand All @@ -684,7 +684,7 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str
:param directory:
The directory with which to interact. This can either be the name of the directory,
or an instance of DirectoryProperties.
:type directory: str or ~azure.storage.filedatalake.DirectoryProperties
:type directory: str or ~azure.storage.filedatalake.DirectoryProperties or a PurePosixPath
:returns: A DataLakeDirectoryClient.
:rtype: ~azure.storage.filedatalake.aio.DataLakeDirectoryClient

Expand All @@ -697,10 +697,13 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str
:dedent: 12
:caption: Getting the directory client to interact with a specific directory.
"""
try:
directory_name = directory.name
except AttributeError:
directory_name = directory
if not isinstance(directory, DirectoryProperties) and hasattr(directory, "name"):
directory_name = str(directory)
else:
try:
directory_name = directory.name
except AttributeError:
directory_name = directory
_pipeline = AsyncPipeline(
transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable = protected-access
policies=self._pipeline._impl_policies # pylint: disable = protected-access
Expand All @@ -715,7 +718,7 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str
loop=self._loop
)

def get_file_client(self, file_path # type: Union[FileProperties, str]
def get_file_client(self, file_path # type: Union[FileProperties, str, PurePosixPath]
):
# type: (...) -> DataLakeFileClient
"""Get a client to interact with the specified file.
Expand All @@ -725,7 +728,7 @@ def get_file_client(self, file_path # type: Union[FileProperties, str]
:param file_path:
The file with which to interact. This can either be the path of the file(from root directory),
or an instance of FileProperties. eg. directory/subdirectory/file
:type file_path: str or ~azure.storage.filedatalake.FileProperties
:type file_path: str or ~azure.storage.filedatalake.FileProperties or a PurePosixPath
:returns: A DataLakeFileClient.
:rtype: ~azure.storage.filedatalake.aio.DataLakeFileClient

Expand All @@ -738,10 +741,13 @@ def get_file_client(self, file_path # type: Union[FileProperties, str]
:dedent: 12
:caption: Getting the file client to interact with a specific file.
"""
try:
file_path = file_path.name
except AttributeError:
pass
if not isinstance(file_path, FileProperties) and hasattr(file_path, "name"):
file_path = str(file_path)
else:
try:
file_path = file_path.name
except AttributeError:
pass
_pipeline = AsyncPipeline(
transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable = protected-access
policies=self._pipeline._impl_policies # pylint: disable = protected-access
Expand Down