From 8fe2f64043ba79c749d21ef5a5d9c103793dc728 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Sun, 6 Oct 2024 13:18:57 +0200 Subject: [PATCH 1/5] Fixed a small bug in 'InfiniteLookBackBuffer' that was causing an error for composite spaces Signed-off-by: simonsays1980 --- rllib/env/utils/infinite_lookback_buffer.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/rllib/env/utils/infinite_lookback_buffer.py b/rllib/env/utils/infinite_lookback_buffer.py index 269e3827ca20..7ec9ff0d4ee9 100644 --- a/rllib/env/utils/infinite_lookback_buffer.py +++ b/rllib/env/utils/infinite_lookback_buffer.py @@ -74,9 +74,6 @@ def get_state(self) -> Dict[str, Any]: "data": self.data, "lookback": self.lookback, "finalized": self.finalized, - "space_struct": gym_space_to_dict(self.space_struct) - if self.space_struct - else self.space_struct, "space": gym_space_to_dict(self.space) if self.space else self.space, } @@ -95,14 +92,12 @@ def from_state(state: Dict[str, Any]) -> None: buffer.data = state["data"] buffer.lookback = state["lookback"] buffer.finalized = state["finalized"] - buffer.space_struct = ( - gym_space_from_dict(state["space_struct"]) - if state["space_struct"] - else state["space_struct"] - ) buffer.space = ( gym_space_from_dict(state["space"]) if state["space"] else state["space"] ) + buffer.space_struct = ( + get_base_struct_from_space(buffer.space) if buffer.space else buffer.space + ) return buffer From a8bb30b8382827f6d6189912135e5b15eee5b2ca Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Mon, 7 Oct 2024 16:25:37 +0200 Subject: [PATCH 2/5] Used 'to_jsonable_if_needed' and 'from_jsonable_if_needed' to convert composite space samples if spaces are given. Signed-off-by: simonsays1980 --- rllib/env/utils/infinite_lookback_buffer.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/rllib/env/utils/infinite_lookback_buffer.py b/rllib/env/utils/infinite_lookback_buffer.py index 7ec9ff0d4ee9..b69f1d6dd65c 100644 --- a/rllib/env/utils/infinite_lookback_buffer.py +++ b/rllib/env/utils/infinite_lookback_buffer.py @@ -8,8 +8,10 @@ from ray.rllib.utils.serialization import gym_space_from_dict, gym_space_to_dict from ray.rllib.utils.spaces.space_utils import ( batch, + from_jsonable_if_needed, get_dummy_batch_for_space, get_base_struct_from_space, + to_jsonable_if_needed, ) @@ -71,9 +73,12 @@ def get_state(self) -> Dict[str, Any]: A dict containing all the data and metadata from the buffer. """ return { - "data": self.data, + "data": to_jsonable_if_needed(self.data, self.space) if self.space else self.data, "lookback": self.lookback, "finalized": self.finalized, + #"space_struct": gym_space_to_dict(self.space_struct) + # if self.space_struct + # else self.space_struct, "space": gym_space_to_dict(self.space) if self.space else self.space, } @@ -89,15 +94,17 @@ def from_state(state: Dict[str, Any]) -> None: from the state dict. """ buffer = InfiniteLookbackBuffer() - buffer.data = state["data"] buffer.lookback = state["lookback"] buffer.finalized = state["finalized"] buffer.space = ( gym_space_from_dict(state["space"]) if state["space"] else state["space"] ) buffer.space_struct = ( - get_base_struct_from_space(buffer.space) if buffer.space else buffer.space + get_base_struct_from_space(buffer.space) + if buffer.space + else buffer.space ) + buffer.data = from_jsonable_if_needed(state["data"], buffer.space) if buffer.space else state["data"] return buffer From fdd4c865debed1336feb6c7bc7077eb6b04a13a2 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Mon, 7 Oct 2024 16:34:21 +0200 Subject: [PATCH 3/5] Used 'to_jsonable_if_needed' and 'from_jsonable_if_needed' to convert composite space samples if spaces are given. Signed-off-by: simonsays1980 --- rllib/env/utils/infinite_lookback_buffer.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/rllib/env/utils/infinite_lookback_buffer.py b/rllib/env/utils/infinite_lookback_buffer.py index b69f1d6dd65c..c48a6c0a8070 100644 --- a/rllib/env/utils/infinite_lookback_buffer.py +++ b/rllib/env/utils/infinite_lookback_buffer.py @@ -73,10 +73,12 @@ def get_state(self) -> Dict[str, Any]: A dict containing all the data and metadata from the buffer. """ return { - "data": to_jsonable_if_needed(self.data, self.space) if self.space else self.data, + "data": to_jsonable_if_needed(self.data, self.space) + if self.space + else self.data, "lookback": self.lookback, "finalized": self.finalized, - #"space_struct": gym_space_to_dict(self.space_struct) + # "space_struct": gym_space_to_dict(self.space_struct) # if self.space_struct # else self.space_struct, "space": gym_space_to_dict(self.space) if self.space else self.space, @@ -100,11 +102,13 @@ def from_state(state: Dict[str, Any]) -> None: gym_space_from_dict(state["space"]) if state["space"] else state["space"] ) buffer.space_struct = ( - get_base_struct_from_space(buffer.space) + get_base_struct_from_space(buffer.space) if buffer.space else buffer.space + ) + buffer.data = ( + from_jsonable_if_needed(state["data"], buffer.space) if buffer.space - else buffer.space + else state["data"] ) - buffer.data = from_jsonable_if_needed(state["data"], buffer.space) if buffer.space else state["data"] return buffer From 15de03d6169c3a37728bd7e69867baebc0d2e344 Mon Sep 17 00:00:00 2001 From: Sven Mika Date: Tue, 8 Oct 2024 19:41:52 +0200 Subject: [PATCH 4/5] Apply suggestions from code review Signed-off-by: Sven Mika --- rllib/env/utils/infinite_lookback_buffer.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rllib/env/utils/infinite_lookback_buffer.py b/rllib/env/utils/infinite_lookback_buffer.py index c48a6c0a8070..66c9e69da0fd 100644 --- a/rllib/env/utils/infinite_lookback_buffer.py +++ b/rllib/env/utils/infinite_lookback_buffer.py @@ -78,9 +78,6 @@ def get_state(self) -> Dict[str, Any]: else self.data, "lookback": self.lookback, "finalized": self.finalized, - # "space_struct": gym_space_to_dict(self.space_struct) - # if self.space_struct - # else self.space_struct, "space": gym_space_to_dict(self.space) if self.space else self.space, } @@ -99,10 +96,10 @@ def from_state(state: Dict[str, Any]) -> None: buffer.lookback = state["lookback"] buffer.finalized = state["finalized"] buffer.space = ( - gym_space_from_dict(state["space"]) if state["space"] else state["space"] + gym_space_from_dict(state["space"]) if state["space"] else None ) buffer.space_struct = ( - get_base_struct_from_space(buffer.space) if buffer.space else buffer.space + get_base_struct_from_space(buffer.space) if buffer.space else None ) buffer.data = ( from_jsonable_if_needed(state["data"], buffer.space) From 51203fe6776966cb3ca1163e28583850ee6f2126 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Wed, 9 Oct 2024 13:59:01 +0200 Subject: [PATCH 5/5] LINTER. Signed-off-by: simonsays1980 --- rllib/env/utils/infinite_lookback_buffer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rllib/env/utils/infinite_lookback_buffer.py b/rllib/env/utils/infinite_lookback_buffer.py index 66c9e69da0fd..cd84a8518097 100644 --- a/rllib/env/utils/infinite_lookback_buffer.py +++ b/rllib/env/utils/infinite_lookback_buffer.py @@ -95,9 +95,7 @@ def from_state(state: Dict[str, Any]) -> None: buffer = InfiniteLookbackBuffer() buffer.lookback = state["lookback"] buffer.finalized = state["finalized"] - buffer.space = ( - gym_space_from_dict(state["space"]) if state["space"] else None - ) + buffer.space = gym_space_from_dict(state["space"]) if state["space"] else None buffer.space_struct = ( get_base_struct_from_space(buffer.space) if buffer.space else None )