diff --git a/.circleci/config.yml b/.circleci/config.yml
index aacd169e76ae..83a22425ddcc 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -136,12 +136,6 @@ jobs:
       - deploy:
           name: Update the docs
           command: nox -e docs
-      - deploy:
-          name: Setup to Push to PyPI
-          command: |
-            if [[ -n "$CIRCLE_TAG" ]]; then
-              pip install --upgrade setuptools twine
-            fi
       - deploy:
           name: Push to PyPI (if this is a release tag).
           command: test_utils/scripts/circleci/twine_upload.sh
diff --git a/test_utils/scripts/circleci/twine_upload.sh b/test_utils/scripts/circleci/twine_upload.sh
index 9bbf3b1b3121..8090065187c8 100755
--- a/test_utils/scripts/circleci/twine_upload.sh
+++ b/test_utils/scripts/circleci/twine_upload.sh
@@ -29,5 +29,7 @@ PKG_DIR="$(python ${SCRIPT})"
 
 # Move into the package, build the distribution and upload.
 cd ${PKG_DIR}
-python setup.py sdist bdist_wheel
+python3 setup.py sdist bdist_wheel
+
+pip3 install twine
 twine upload dist/*