feat(off-policy): fix final_obsevation setting and support evaluation…

… times configuation (#260)

omnisafe/adapter/offpolicy_adapter.py

@@ -137,7 +137,8 @@ def rollout(  # pylint: disable=too-many-locals
             real_next_obs = next_obs.clone()
             for idx, done in enumerate(torch.logical_or(terminated, truncated)):
                 if done:
-                    real_next_obs[idx] = info['final_observation'][idx]
+                    if 'final_observation' in info:
+                        real_next_obs[idx] = info['final_observation'][idx]
                     self._log_metrics(logger, idx)
                     self._reset_log(idx)
 

omnisafe/adapter/online_adapter.py

@@ -72,7 +72,6 @@ def __init__(  # pylint: disable=too-many-arguments
         )
 
         self._env.set_seed(seed)
-        self._eval_env.set_seed(seed)
 
     def _wrapper(
         self,

omnisafe/algorithms/off_policy/ddpg.py

@@ -200,9 +200,10 @@ def _init_log(self) -> None:
         self._logger.register_key('Metrics/EpCost', window_length=50)
         self._logger.register_key('Metrics/EpLen', window_length=50)
 
-        self._logger.register_key('Metrics/TestEpRet', window_length=50)
-        self._logger.register_key('Metrics/TestEpCost', window_length=50)
-        self._logger.register_key('Metrics/TestEpLen', window_length=50)
+        if self._cfgs.train_cfgs.eval_episodes > 0:
+            self._logger.register_key('Metrics/TestEpRet', window_length=50)
+            self._logger.register_key('Metrics/TestEpCost', window_length=50)
+            self._logger.register_key('Metrics/TestEpLen', window_length=50)
 
         self._logger.register_key('Train/Epoch')
         self._logger.register_key('Train/LR')
@@ -283,7 +284,7 @@ def learn(self) -> tuple[float, float, float]:
 
             eval_start = time.time()
             self._env.eval_policy(
-                episode=1,
+                episode=self._cfgs.train_cfgs.eval_episodes,
                 agent=self._actor_critic,
                 logger=self._logger,
             )

omnisafe/configs/off-policy/DDPG.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy

omnisafe/configs/off-policy/DDPGLag.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy

omnisafe/configs/off-policy/DDPGPID.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy

omnisafe/configs/off-policy/SAC.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy

omnisafe/configs/off-policy/SACLag.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy

omnisafe/configs/off-policy/SACPID.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy

omnisafe/configs/off-policy/TD3.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy

omnisafe/configs/off-policy/TD3Lag.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy

omnisafe/configs/off-policy/TD3PID.yaml

@@ -28,6 +28,8 @@ defaults:
     parallel: 1
     # total number of steps to train
     total_steps: 1000000
+    # number of evaluate episodes
+    eval_episodes: 1
   # algorithm configurations
   algo_cfgs:
     # number of steps to update the policy