diff --git a/rllib/tuned_examples/apex_dqn/atari-apex-dqn.yaml b/rllib/tuned_examples/apex_dqn/atari-apex-dqn.yaml
index 3ab1e251bb99..d858500662c4 100644
--- a/rllib/tuned_examples/apex_dqn/atari-apex-dqn.yaml
+++ b/rllib/tuned_examples/apex_dqn/atari-apex-dqn.yaml
@@ -27,4 +27,4 @@ apex-breakoutnoframeskip-v4:
         rollout_fragment_length: 20
         train_batch_size: 512
         target_network_update_freq: 50000
-        min_sample_timesteps_per_iteration: 25000
\ No newline at end of file
+        min_sample_timesteps_per_iteration: 25000
diff --git a/rllib/tuned_examples/apex_dqn/pong-apex-dqn.yaml b/rllib/tuned_examples/apex_dqn/pong-apex-dqn.yaml
index 25c483ec5380..97eed96b1024 100644
--- a/rllib/tuned_examples/apex_dqn/pong-apex-dqn.yaml
+++ b/rllib/tuned_examples/apex_dqn/pong-apex-dqn.yaml
@@ -18,5 +18,8 @@ pong-apex:
         replay_buffer_config:
           type: MultiAgentPrioritizedReplayBuffer
           capacity: 1000000
+        # we should set compress_observations to True because few machines
+        # would be able to contain the replay buffers in memory otherwise
+        compress_observations: True
         gamma: 0.99
         training_intensity: 16
diff --git a/rllib/tuned_examples/dqn/pong-dqn.yaml b/rllib/tuned_examples/dqn/pong-dqn.yaml
index 368d1308a29f..1b7cc565e91c 100644
--- a/rllib/tuned_examples/dqn/pong-dqn.yaml
+++ b/rllib/tuned_examples/dqn/pong-dqn.yaml
@@ -24,3 +24,6 @@ pong-deterministic-dqn:
           grayscale: True
           zero_mean: False
           dim: 42
+        # we should set compress_observations to True because few machines
+        # would be able to contain the replay buffers in memory otherwise
+        compress_observations: True
diff --git a/rllib/tuned_examples/dqn/pong-rainbow.yaml b/rllib/tuned_examples/dqn/pong-rainbow.yaml
index 1c2f0720fd5d..5264ebb882dc 100644
--- a/rllib/tuned_examples/dqn/pong-rainbow.yaml
+++ b/rllib/tuned_examples/dqn/pong-rainbow.yaml
@@ -26,3 +26,6 @@ pong-deterministic-rainbow:
           grayscale: True
           zero_mean: False
           dim: 42
+        # we should set compress_observations to True because few machines
+        # would be able to contain the replay buffers in memory otherwise
+        compress_observations: True