From f29783d0a0ab9ec66f9dd9ae8ae7840dac8c86b4 Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Mon, 19 Aug 2024 10:50:56 -0700 Subject: [PATCH 1/2] Add docstring to explain Dataset.deserialize_lineage Signed-off-by: Cheng Su --- python/ray/data/dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 08bbed3571b9..3bb7dd3a8262 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -4777,8 +4777,8 @@ def serialize_lineage(self) -> bytes: futures, to bytes that can be stored and later deserialized, possibly on a different cluster. - Note that this will drop all computed data, and that everything is - recomputed from scratch after deserialization. + Note that this use pickle and will drop all computed data, and that everything + is recomputed from scratch after deserialization. Use :py:meth:`Dataset.deserialize_lineage` to deserialize the serialized bytes returned from this method into a Dataset. @@ -4866,8 +4866,8 @@ def deserialize_lineage(serialized_ds: bytes) -> "Dataset": """ Deserialize the provided lineage-serialized Dataset. - This assumes that the provided serialized bytes were serialized using - :py:meth:`Dataset.serialize_lineage`. + This uses pickle, and assumes that the provided serialized bytes were + serialized using :py:meth:`Dataset.serialize_lineage`. Examples: From 22f78b66126f1164eaa2dd548c19fad7cb0f6273 Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Mon, 19 Aug 2024 11:17:28 -0700 Subject: [PATCH 2/2] Update python/ray/data/dataset.py Co-authored-by: Scott Lee Signed-off-by: Cheng Su --- python/ray/data/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 3bb7dd3a8262..eafaeb3a60fa 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -4777,7 +4777,7 @@ def serialize_lineage(self) -> bytes: futures, to bytes that can be stored and later deserialized, possibly on a different cluster. - Note that this use pickle and will drop all computed data, and that everything + Note that this uses pickle and will drop all computed data, and that everything is recomputed from scratch after deserialization. Use :py:meth:`Dataset.deserialize_lineage` to deserialize the serialized