[RLlib] Update bandit_envs_recommender_system

rllib/env/wrappers/recsim.py

@@ -84,25 +84,20 @@ def __init__(self, env: gym.Env):
         self.observation_space = Dict(
             OrderedDict(
                 [
-                    ("user", obs_space["user"]),
                     (
                         "item",
                         gym.spaces.Box(
-                            low=-np.ones((num_items, embedding_dim)),
-                            high=np.ones((num_items, embedding_dim)),
+                            low=-1.0, high=1.0, shape=(num_items, embedding_dim)
                         ),
                     ),
-                    ("response", obs_space["response"]),
                 ]
             )
         )
         self._sampled_obs = self.observation_space.sample()
 
     def observation(self, obs):
         new_obs = OrderedDict()
-        new_obs["user"] = obs["user"]
         new_obs["item"] = np.vstack(list(obs["doc"].values()))
-        new_obs["response"] = obs["response"]
         new_obs = convert_element_to_space_type(new_obs, self._sampled_obs)
         return new_obs
 

rllib/examples/bandit/tune_lin_ucb_train_recommendation.py

@@ -8,7 +8,23 @@
 
 import ray
 from ray import tune
-from ray.rllib.examples.env.bandit_envs_recommender_system import ParametricItemRecoEnv
+from ray.tune import register_env
+from ray.rllib.env.wrappers.recsim import (
+    MultiDiscreteToDiscreteActionWrapper,
+    RecSimObservationBanditWrapper,
+)
+from ray.rllib.examples.env.bandit_envs_recommender_system import (
+    ParametricRecSys,
+)
+
+# Because ParametricRecSys follows RecSim's API, we have to wrap it before
+# it can work with our Bandits agent.
+register_env(
+    "ParametricRecSysEnv",
+    lambda cfg: MultiDiscreteToDiscreteActionWrapper(
+        RecSimObservationBanditWrapper(ParametricRecSys(**cfg))
+    ),
+)
 
 if __name__ == "__main__":
     # Temp fix to avoid OMP conflict.
@@ -17,8 +33,20 @@
     ray.init()
 
     config = {
-        "env": ParametricItemRecoEnv,
+        "env": "ParametricRecSysEnv",
+        "env_config": {
+            "embedding_size": 20,
+            "num_docs_to_select_from": 10,
+            "slate_size": 1,
+            "num_docs_in_db": 100,
+            "num_users_in_db": 1,
+            "user_time_budget": 1.0,
+        },
         "num_envs_per_worker": 2,  # Test with batched inference.
+        "evaluation_interval": 20,
+        "evaluation_duration": 100,
+        "evaluation_duration_unit": "episodes",
+        "simple_optimizer": True,
     }
 
     # Actual training_iterations will be 10 * timesteps_per_iteration

rllib/examples/bandit/tune_lin_ucb_train_recsim_env.py

@@ -21,14 +21,16 @@
         # Then: "env": [the imported RecSim class]
         "env": "RecSim-v1",
         "env_config": {
+            "num_candidates": 10,
+            "slate_size": 1,
             "convert_to_discrete_action_space": True,
             "wrap_for_bandits": True,
         },
     }
 
     # Actual training_iterations will be 10 * timesteps_per_iteration
-    # (100 by default) = 2,000
-    training_iterations = 10
+    # (100 by default) = 500,000
+    training_iterations = 5000
 
     print("Running training for %s time steps" % training_iterations)