diff --git a/.github/workflows/hvd-tests.yml b/.github/workflows/hvd-tests.yml
new file mode 100644
index 00000000000..3fdc4ef0d21
--- /dev/null
+++ b/.github/workflows/hvd-tests.yml
@@ -0,0 +1,36 @@
+name: Run Horovod tests
+on: [push, pull_request]
+
+jobs:
+  horovod-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7]
+        pytorch-channel: [pytorch]
+      
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Miniconda
+      uses: goanpeca/setup-miniconda@v1
+      with:
+        miniconda-version: "latest"
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      shell: bash -l {0}
+      run: |      
+        #install other dependencies
+        conda install pytorch torchvision cpuonly -c ${{ matrix.pytorch-channel }}
+        pip install -r requirements-dev.txt
+        # Fixes #1153
+        pip install --upgrade scipy==1.4.1
+        pip install horovod
+        python setup.py install
+      
+    - name: Run Tests
+      shell: bash -l {0}
+      run: |
+        SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} CI_PYTHON_VERSION="${{ matrix.python-version }}" sh tests/run_cpu_tests.sh    
+