diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py index d6b480784b15..e5973e85ff4a 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py @@ -305,7 +305,7 @@ def upload_data(self, data, # type: Union[AnyStr, Iterable[AnyStr], IO[AnyStr]] :keyword ~azure.storage.filedatalake.DataLakeLeaseClient or str lease: Required if the blob has an active lease. Value can be a DataLakeLeaseClient object or the lease ID as a string. - :keyword str umaskoverwrite: Optional and only valid if Hierarchical Namespace is enabled for the account. + :keyword str umask: Optional and only valid if Hierarchical Namespace is enabled for the account. When creating a file or directory and the parent folder does not have a default ACL, the umask restricts the permissions of the file or directory to be created. The resulting permission is given by p & ^u, where p is the permission and u is the umask. diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_upload_helper.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_upload_helper.py index e1db768eff3b..bf29cfc250a7 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_upload_helper.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_upload_helper.py @@ -37,6 +37,8 @@ def upload_datalake_file( # pylint: disable=unused-argument if length == 0: return {} properties = kwargs.pop('properties', None) + umask = kwargs.pop('umask', None) + permissions = kwargs.pop('permissions', None) path_http_headers = kwargs.pop('path_http_headers', None) modified_access_conditions = kwargs.pop('modified_access_conditions', None) @@ -44,8 +46,8 @@ def upload_datalake_file( # pylint: disable=unused-argument # if customers didn't specify access conditions, they cannot flush data to existing file if not _any_conditions(modified_access_conditions): modified_access_conditions.if_none_match = '*' - if properties: - raise ValueError("metadata can be set only when overwrite is enabled") + if properties or umask or permissions: + raise ValueError("metadata, umask and permissions can be set only when overwrite is enabled") if overwrite: response = client.create( @@ -53,6 +55,8 @@ def upload_datalake_file( # pylint: disable=unused-argument path_http_headers=path_http_headers, properties=properties, modified_access_conditions=modified_access_conditions, + umask=umask, + permissions=permissions, cls=return_response_headers, **kwargs) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_upload_helper.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_upload_helper.py index caa678b7db64..b2f10df34ec3 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_upload_helper.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_upload_helper.py @@ -37,6 +37,8 @@ async def upload_datalake_file( # pylint: disable=unused-argument if length == 0: return {} properties = kwargs.pop('properties', None) + umask = kwargs.pop('umask', None) + permissions = kwargs.pop('permissions', None) path_http_headers = kwargs.pop('path_http_headers', None) modified_access_conditions = kwargs.pop('modified_access_conditions', None) @@ -44,8 +46,8 @@ async def upload_datalake_file( # pylint: disable=unused-argument # if customers didn't specify access conditions, they cannot flush data to existing file if not _any_conditions(modified_access_conditions): modified_access_conditions.if_none_match = '*' - if properties: - raise ValueError("metadata can be set only when overwrite is enabled") + if properties or umask or permissions: + raise ValueError("metadata, umask and permissions can be set only when overwrite is enabled") if overwrite: response = await client.create( @@ -53,6 +55,8 @@ async def upload_datalake_file( # pylint: disable=unused-argument path_http_headers=path_http_headers, properties=properties, modified_access_conditions=modified_access_conditions, + umask=umask, + permissions=permissions, cls=return_response_headers, **kwargs) diff --git a/sdk/storage/azure-storage-file-datalake/tests/recordings/test_file.test_upload_data_to_existing_file_with_permission_and_umask.yaml b/sdk/storage/azure-storage-file-datalake/tests/recordings/test_file.test_upload_data_to_existing_file_with_permission_and_umask.yaml new file mode 100644 index 000000000000..76bc93cac306 --- /dev/null +++ b/sdk/storage/azure-storage-file-datalake/tests/recordings/test_file.test_upload_data_to_existing_file_with_permission_and_umask.yaml @@ -0,0 +1,336 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - azsdk-python-storage-dfs/12.0.1 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-client-request-id: + - bab07534-8468-11ea-a05e-001a7dda7113 + x-ms-date: + - Wed, 22 Apr 2020 07:13:13 GMT + x-ms-properties: + - '' + x-ms-version: + - '2019-02-02' + method: PUT + uri: https://storagename.dfs.core.windows.net/filesystemde181c6b/directoryde181c6b?resource=directory + response: + body: + string: '' + headers: + Content-Length: + - '0' + Date: + - Wed, 22 Apr 2020 07:13:13 GMT + ETag: + - '"0x8D7E68C9F3C17C1"' + Last-Modified: + - Wed, 22 Apr 2020 07:13:13 GMT + Server: + - Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0 + x-ms-request-id: + - 4c6f11c3-c01f-0010-0775-1888fc000000 + x-ms-version: + - '2019-02-02' + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - azsdk-python-storage-dfs/12.0.1 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-client-request-id: + - bb1211cc-8468-11ea-bc86-001a7dda7113 + x-ms-date: + - Wed, 22 Apr 2020 07:13:13 GMT + x-ms-properties: + - '' + x-ms-version: + - '2019-02-02' + method: PUT + uri: https://storagename.dfs.core.windows.net/filesystemde181c6b/directoryde181c6b%2Ffilename?resource=file + response: + body: + string: '' + headers: + Content-Length: + - '0' + Date: + - Wed, 22 Apr 2020 07:13:13 GMT + ETag: + - '"0x8D7E68C9F50D057"' + Last-Modified: + - Wed, 22 Apr 2020 07:13:13 GMT + Server: + - Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0 + x-ms-request-id: + - 4c6f11d5-c01f-0010-1875-1888fc000000 + x-ms-version: + - '2019-02-02' + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + If-Match: + - '"0x8D7E68C9F50D057"' + User-Agent: + - azsdk-python-storage-dfs/12.0.1 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-client-request-id: + - bb268580-8468-11ea-82be-001a7dda7113 + x-ms-date: + - Wed, 22 Apr 2020 07:13:13 GMT + x-ms-permissions: + - '0777' + x-ms-properties: + - '' + x-ms-umask: + - '0000' + x-ms-version: + - '2019-02-02' + method: PUT + uri: https://storagename.dfs.core.windows.net/filesystemde181c6b/directoryde181c6b%2Ffilename?resource=file + response: + body: + string: '' + headers: + Content-Length: + - '0' + Date: + - Wed, 22 Apr 2020 07:13:13 GMT + ETag: + - '"0x8D7E68C9F5DCB55"' + Last-Modified: + - Wed, 22 Apr 2020 07:13:13 GMT + Server: + - Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0 + x-ms-request-id: + - 4c6f11e0-c01f-0010-2375-1888fc000000 + x-ms-version: + - '2019-02-02' + status: + code: 201 + message: Created +- request: + body: !!binary | + j08hau8rxoygBJTJ+9qreKaMaINmVz8Nobt31ZNQFoTi6DbehovoxuSsQnAlpTvL8U4V45wuPg3v + nW0+hZ3T03qxzymBYJh+QrZhgmtgO3s9usYaubSji0DugB77R02cG0Zf7g== + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '100' + Content-Type: + - application/json; charset=utf-8 + User-Agent: + - azsdk-python-storage-dfs/12.0.1 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-client-request-id: + - bb3452c0-8468-11ea-a624-001a7dda7113 + x-ms-date: + - Wed, 22 Apr 2020 07:13:14 GMT + x-ms-version: + - '2019-02-02' + method: PATCH + uri: https://storagename.dfs.core.windows.net/filesystemde181c6b/directoryde181c6b%2Ffilename?position=0&action=append + response: + body: + string: '' + headers: + Content-Length: + - '0' + Date: + - Wed, 22 Apr 2020 07:13:13 GMT + Server: + - Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0 + x-ms-request-id: + - 4c6f11e9-c01f-0010-2c75-1888fc000000 + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2019-02-02' + status: + code: 202 + message: Accepted +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + If-Match: + - '"0x8D7E68C9F5DCB55"' + User-Agent: + - azsdk-python-storage-dfs/12.0.1 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-client-request-id: + - bb45abae-8468-11ea-b764-001a7dda7113 + x-ms-date: + - Wed, 22 Apr 2020 07:13:14 GMT + x-ms-version: + - '2019-02-02' + method: PATCH + uri: https://storagename.dfs.core.windows.net/filesystemde181c6b/directoryde181c6b%2Ffilename?position=100&action=flush + response: + body: + string: '' + headers: + Content-Length: + - '0' + Date: + - Wed, 22 Apr 2020 07:13:13 GMT + ETag: + - '"0x8D7E68C9F7DDA34"' + Last-Modified: + - Wed, 22 Apr 2020 07:13:14 GMT + Server: + - Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0 + x-ms-request-id: + - 4c6f11f1-c01f-0010-3475-1888fc000000 + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2019-02-02' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-dfs/12.0.1 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-client-request-id: + - bb53abc8-8468-11ea-9353-001a7dda7113 + x-ms-date: + - Wed, 22 Apr 2020 07:13:14 GMT + x-ms-range: + - bytes=0-33554431 + x-ms-version: + - '2019-07-07' + method: GET + uri: https://storagename.blob.core.windows.net/filesystemde181c6b/directoryde181c6b/filename + response: + body: + string: !!binary | + j08hau8rxoygBJTJ+9qreKaMaINmVz8Nobt31ZNQFoTi6DbehovoxuSsQnAlpTvL8U4V45wuPg3v + nW0+hZ3T03qxzymBYJh+QrZhgmtgO3s9usYaubSji0DugB77R02cG0Zf7g== + headers: + Accept-Ranges: + - bytes + Content-Length: + - '100' + Content-Range: + - bytes 0-99/100 + Content-Type: + - application/octet-stream + Date: + - Wed, 22 Apr 2020 07:13:13 GMT + ETag: + - '"0x8D7E68C9F7DDA34"' + Last-Modified: + - Wed, 22 Apr 2020 07:13:14 GMT + Server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Wed, 22 Apr 2020 07:13:13 GMT + x-ms-lease-state: + - available + x-ms-lease-status: + - unlocked + x-ms-request-id: + - fabf4bb7-001e-000f-4375-183bf8000000 + x-ms-server-encrypted: + - 'true' + x-ms-version: + - '2019-07-07' + status: + code: 206 + message: Partial Content +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-dfs/12.0.1 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-client-request-id: + - bb82e2ca-8468-11ea-973f-001a7dda7113 + x-ms-date: + - Wed, 22 Apr 2020 07:13:14 GMT + x-ms-version: + - '2019-02-02' + method: HEAD + uri: https://storagename.dfs.core.windows.net/filesystemde181c6b/directoryde181c6b%2Ffilename?action=getAccessControl&upn=false + response: + body: + string: '' + headers: + Date: + - Wed, 22 Apr 2020 07:13:14 GMT + ETag: + - '"0x8D7E68C9F7DDA34"' + Last-Modified: + - Wed, 22 Apr 2020 07:13:14 GMT + Server: + - Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0 + x-ms-acl: + - user::rwx,group::rwx,other::rwx + x-ms-group: + - $superuser + x-ms-owner: + - $superuser + x-ms-permissions: + - rwxrwxrwx + x-ms-request-id: + - 4c6f120d-c01f-0010-5075-1888fc000000 + x-ms-version: + - '2019-02-02' + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_file.py b/sdk/storage/azure-storage-file-datalake/tests/test_file.py index dfa7c6000619..bf831b563b66 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_file.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_file.py @@ -291,6 +291,33 @@ def test_upload_data_to_existing_file_with_content_settings(self): self.assertEqual(data, downloaded_data) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) + @record + def test_upload_data_to_existing_file_with_permission_and_umask(self): + directory_name = self._get_directory_reference() + + # Create a directory to put the file under that + directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) + directory_client.create_directory() + + # create an existing file + file_client = directory_client.get_file_client('filename') + etag = file_client.create_file()['etag'] + + # to override the existing file + data = self.get_random_bytes(100) + + file_client.upload_data(data, overwrite=True, max_concurrency=5, + permissions='0777', umask="0000", + etag=etag, + match_condition=MatchConditions.IfNotModified) + + downloaded_data = file_client.download_file().readall() + prop = file_client.get_access_control() + + # Assert + self.assertEqual(data, downloaded_data) + self.assertEqual(prop['permissions'], 'rwxrwxrwx') + @record def test_read_file(self): file_client = self._create_file_and_return_client() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_file_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_file_async.py index 7c70ac124f2d..8bd1ef819a14 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_file_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_file_async.py @@ -350,6 +350,41 @@ def test_upload_data_to_existing_file_with_content_settings_async(self): loop = asyncio.get_event_loop() loop.run_until_complete(self._test_upload_data_to_existing_file_with_content_settings_async()) + async def _test_upload_data_to_existing_file_with_permissions_and_umask_async(self): + # etag in async recording file cannot be parsed properly + if TestMode.need_recording_file(self.test_mode): + return + directory_name = self._get_directory_reference() + + # Create a directory to put the file under that + directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) + await directory_client.create_directory() + + # create an existing file + file_client = directory_client.get_file_client('filename') + resp = await file_client.create_file() + etag = resp['etag'] + + # to override the existing file + data = self.get_random_bytes(100) + + await file_client.upload_data(data, + overwrite=True, max_concurrency=5, + permissions='0777', umask="0000", + etag=etag, + match_condition=MatchConditions.IfNotModified) + + downloaded_data = await (await file_client.download_file()).readall() + prop = await file_client.get_access_control() + + self.assertEqual(data, downloaded_data) + self.assertEqual(prop['permissions'], 'rwxrwxrwx') + + @record + def test_upload_data_to_existing_file_with_permission_and_umask_async(self): + loop = asyncio.get_event_loop() + loop.run_until_complete(self._test_upload_data_to_existing_file_with_permissions_and_umask_async()) + async def _test_read_file(self): file_client = await self._create_file_and_return_client() data = self.get_random_bytes(1024)