Check for the whole protocol including delimited (<scheme>://) when j… (

#2715) * Check for the whole protocol including delimited (<scheme>://) when joining path for partitions in partitioned data set Signed-off-by: Julius Hetzel <[email protected]> * Add bugfix description to release nodes Signed-off-by: Julius Hetzel <[email protected]> --------- Signed-off-by: Julius Hetzel <[email protected]>
kedro-org · Jun 23, 2023 · fd8162d · fd8162d
1 parent 160fd6b
commit fd8162d
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 4 deletions.
diff --git a/RELEASE.md b/RELEASE.md
@@ -3,6 +3,7 @@
 ## Major features and improvements
 
 ## Bug fixes and other changes
+* Compare for protocol and delimiter in `PartitionedDataSet` to be able to pass the protocol to partitions which paths starts with the same characters as the protocol (e.g. `s3://s3-my-bucket`).
 
 ## Breaking changes to the API
 

diff --git a/kedro/io/partitioned_dataset.py b/kedro/io/partitioned_dataset.py
@@ -263,10 +263,11 @@ def _list_partitions(self) -> list[str]:
         ]
 
     def _join_protocol(self, path: str) -> str:
-        if self._path.startswith(self._protocol) and not path.startswith(
-            self._protocol
+        protocol_prefix = f"{self._protocol}://"
+        if self._path.startswith(protocol_prefix) and not path.startswith(
+            protocol_prefix
         ):
-            return f"{self._protocol}://{path}"
+            return f"{protocol_prefix}{path}"
         return path
 
     def _partition_to_path(self, path: str):

diff --git a/tests/io/test_partitioned_dataset.py b/tests/io/test_partitioned_dataset.py
@@ -401,7 +401,7 @@ def test_dataset_creds(self, pds_config, expected_ds_creds, global_creds):
         assert pds._credentials == global_creds
 
 
-BUCKET_NAME = "fake_bucket_name"
+BUCKET_NAME = "s3_fake_bucket_name"
 S3_DATASET_DEFINITION = [
     "pandas.CSVDataSet",
     "kedro.extras.datasets.pandas.CSVDataSet",