diff --git a/rllib/BUILD b/rllib/BUILD
index b881f4f104a2..b566b53633ce 100644
--- a/rllib/BUILD
+++ b/rllib/BUILD
@@ -711,7 +711,8 @@ py_test(
     name = "test_algorithm",
     tags = ["team:rllib", "algorithms_dir", "algorithms_dir_generic"],
     size = "large",
-    srcs = ["algorithms/tests/test_algorithm.py"]
+    srcs = ["algorithms/tests/test_algorithm.py"],
+    data = ["tests/data/cartpole/small.json"],
 )
 
 py_test(
@@ -825,6 +826,7 @@ py_test(
     name = "test_cql",
     tags = ["team:rllib", "algorithms_dir"],
     size = "large",
+    data = ["tests/data/cartpole/small.json"],
     srcs = ["algorithms/cql/tests/test_cql.py"]
 )
 
@@ -833,7 +835,8 @@ py_test(
     name = "test_crr",
     tags = ["team:rllib", "algorithms_dir"],
     size = "medium",
-    srcs = ["algorithms/crr/tests/test_crr.py"]
+    srcs = ["algorithms/crr/tests/test_crr.py"],
+    data = ["tests/data/pendulum/large.json"],
 )
 
 # DDPG
@@ -896,7 +899,11 @@ py_test(
     tags = ["team:rllib", "algorithms_dir"],
     size = "large",
     # Include the json data file.
-    data = ["tests/data/cartpole/large.json"],
+    data = [
+        "tests/data/cartpole/large.json",
+        "tests/data/pendulum/large.json",
+        "tests/data/cartpole/small.json",
+    ],
     srcs = ["algorithms/marwil/tests/test_marwil.py"]
 )
 
@@ -1651,7 +1658,8 @@ py_test(
     name = "test_dataset_reader",
     tags = ["team:rllib", "offline"],
     size = "small",
-    srcs = ["offline/tests/test_dataset_reader.py"]
+    srcs = ["offline/tests/test_dataset_reader.py"],
+    data = ["tests/data/pendulum/large.json"],
 )
 
 py_test(
@@ -1665,14 +1673,16 @@ py_test(
     name = "test_json_reader",
     tags = ["team:rllib", "offline"],
     size = "small",
-    srcs = ["offline/tests/test_json_reader.py"]
+    srcs = ["offline/tests/test_json_reader.py"],
+    data = ["tests/data/pendulum/large.json"],
 )
 
 py_test(
     name = "test_ope",
     tags = ["team:rllib", "offline", "torch_only"],
     size = "medium",
-    srcs = ["offline/estimators/tests/test_ope.py"]
+    srcs = ["offline/estimators/tests/test_ope.py"],
+    data = ["tests/data/cartpole/large.json"],
 )