diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh
index 8ab2c6112ce7..d5df927e32b9 100755
--- a/ci/travis/install-dependencies.sh
+++ b/ci/travis/install-dependencies.sh
@@ -309,13 +309,8 @@ install_dependencies() {
   if [ -n "${PYTHON-}" ] && [ "${MINIMAL_INSTALL-}" != 1 ]; then
     # Remove this entire section once Serve dependencies are fixed.
     if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TRAIN_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ] && [ "${RLLIB_TESTING-}" != 1 ]; then
-      # PyTorch is installed first since we are using a "-f" directive to find the wheels.
       # We want to install the CPU version only.
-      local torch_url="https://download.pytorch.org/whl/torch_stable.html"
-      case "${OSTYPE}" in
-        darwin*) pip install torch torchvision;;
-        *) pip install torch==1.9.0+cpu torchvision==0.10.0+cpu -f "${torch_url}";;
-      esac
+      pip install -r "${WORKSPACE_DIR}"/python/requirements/ml/requirements_dl.txt
     fi
 
     # Try n times; we often encounter OpenSSL.SSL.WantReadError (or others)
diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 197adbe4ffbe..a8171b4ce24e 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -19,20 +19,26 @@ RUN sudo apt-get update \
         libgl1-mesa-dev \
         unzip \
         unrar \
-    && $HOME/anaconda3/bin/pip install -U pip \
-    && $HOME/anaconda3/bin/pip --use-deprecated=legacy-resolver --no-cache-dir install -r requirements.txt \
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U -r requirements_rllib.txt \
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U -r requirements_train.txt \
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U -r requirements_tune.txt \
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U -r requirements_ml_docker.txt \
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U -r requirements_upstream.txt \
+    && $HOME/anaconda3/bin/pip --no-cache-dir install -U pip \
+    && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
+           -r requirements.txt \
+           -r requirements_rllib.txt \
+           -r requirements_train.txt \
+           -r requirements_tune.txt \
+           -r requirements_upstream.txt \
+    # explicitly install (overwrite) pytorch with CUDA support
+    && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
+           -r requirements_ml_docker.txt \
     # Remove dataclasses & typing because they are included in Python > 3.6
     && if [ $(python -c 'import sys; print(sys.version_info.minor)') != "6" ]; then \
         $HOME/anaconda3/bin/pip uninstall dataclasses typing -y; fi  \
-    && sudo rm requirements.txt && sudo rm requirements_ml_docker.txt \
-    && sudo rm requirements_tune.txt && sudo rm requirements_rllib.txt \
-    && sudo rm requirements_train.txt \
-    && sudo rm requirements_upstream.txt \
+    && sudo rm \
+        requirements.txt \
+        requirements_ml_docker.txt \
+        requirements_tune.txt \
+        requirements_rllib.txt \
+        requirements_train.txt \
+        requirements_upstream.txt \
     && sudo apt-get clean
 
 # Make sure tfp is installed correctly and matches tf version.
diff --git a/python/requirements.txt b/python/requirements.txt
index 7f772288ff9b..30fa383983dc 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -85,7 +85,6 @@ pytest-lazy-fixture
 pytest-timeout
 pytest-virtualenv
 scikit-learn==0.22.2
-tensorflow==2.5.1
 testfixtures
 werkzeug
 xlrd
diff --git a/python/requirements/ml/requirements_dl.txt b/python/requirements/ml/requirements_dl.txt
index 3970fa10a975..0d833c2b312d 100644
--- a/python/requirements/ml/requirements_dl.txt
+++ b/python/requirements/ml/requirements_dl.txt
@@ -1,14 +1,10 @@
 # These requirements are used for the CI and CPU-only Docker images so we install CPU only versions of torch.
-# For GPU Docker images, you should install requirements_ml_docker.txt instead.
+# For GPU Docker images, you should install requirements_ml_docker.txt afterwards.
 
+tensorflow==2.6.2
+tensorflow-probability==0.14.1
 
-# If you make changes to this file, please also change `requirements_ml_docker.txt`!
-
-
-# TODO(amogkam): Remove after https://github.com/tensorflow/tensorflow/issues/52922 is fixed.
-keras==2.6.0
-tensorflow==2.6.0
-tensorflow-probability==0.14.0
+# If you make changes below this line, please also change `requirements_ml_docker.txt`!
 
 torch==1.9.0;sys_platform=="darwin"
 torchvision==0.10.0;sys_platform=="darwin"
@@ -16,5 +12,4 @@ torchvision==0.10.0;sys_platform=="darwin"
 # On non-OSX machines only install CPU version of torch and torchvision
 -f https://download.pytorch.org/whl/torch_stable.html
 torch==1.9.0+cpu;sys_platform!="darwin"
--f https://download.pytorch.org/whl/torch_stable.html
-torchvision==0.10.0+cpu;sys_platform!="darwin"
\ No newline at end of file
+torchvision==0.10.0+cpu;sys_platform!="darwin"
diff --git a/python/requirements/ml/requirements_rllib.txt b/python/requirements/ml/requirements_rllib.txt
index 2d23f7c1e70e..65ab046456e2 100644
--- a/python/requirements/ml/requirements_rllib.txt
+++ b/python/requirements/ml/requirements_rllib.txt
@@ -31,8 +31,6 @@ higher==0.2.1
 # For auto-generating an env-rendering Window.
 pyglet==1.5.15
 imageio-ffmpeg==0.4.5
-# For JSON reader/writer.
-smart_open==5.0.0
 # Ray Serve example
 starlette==0.16.0
 # ONNX
diff --git a/python/requirements/ml/requirements_tune.txt b/python/requirements/ml/requirements_tune.txt
index 6c1336ae1c1c..fc2edf1c88d2 100644
--- a/python/requirements/ml/requirements_tune.txt
+++ b/python/requirements/ml/requirements_tune.txt
@@ -33,7 +33,6 @@ shortuuid==1.0.1
 scikit-learn==0.24.2
 scikit-optimize==0.8.1
 sigopt==7.5.0
-smart_open==5.1.0
 timm==0.4.5
 transformers==4.10.0
 wandb==0.12.5
diff --git a/python/requirements_ml_docker.txt b/python/requirements_ml_docker.txt
index 166d2bab9e32..1647c81bc76b 100644
--- a/python/requirements_ml_docker.txt
+++ b/python/requirements_ml_docker.txt
@@ -5,13 +5,6 @@ tblib
 
 # If you make changes to anything below this line, please also change `requirements_dl.txt`!
 
-# TODO(amogkam): Remove after https://github.com/tensorflow/tensorflow/issues/52922 is fixed.
-keras==2.6.0
-# In TF >v2, GPU support is included in the base package.
-tensorflow==2.6.0
-tensorflow-probability==0.14.0
-
 -f https://download.pytorch.org/whl/torch_stable.html
 torch==1.9.0+cu111
--f https://download.pytorch.org/whl/torch_stable.html
 torchvision==0.10.0+cu111