diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
index fc19a34b47..f2add6d6a0 100644
--- a/tests/ExtraTests/CMakeLists.txt
+++ b/tests/ExtraTests/CMakeLists.txt
@@ -14,6 +14,15 @@ add_test(
   COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testSharding.py $<TARGET_FILE:SelfTest>
 )
 
+add_test(
+  NAME TestSharding::OverlyLargeShardIndex
+  COMMAND $<TARGET_FILE:SelfTest>  --shard-index 5 --shard-count 5
+)
+set_tests_properties(
+    TestSharding::OverlyLargeShardIndex
+  PROPERTIES
+    PASS_REGULAR_EXPRESSION "The shard count \\(5\\) must be greater than the shard index \\(5\\)"
+)
 
 # The MinDuration reporting tests do not need separate compilation, but
 # they have non-trivial execution time, so they are categorized as
diff --git a/tests/TestScripts/testSharding.py b/tests/TestScripts/testSharding.py
index 9eef3ec038..fcea03e2e1 100644
--- a/tests/TestScripts/testSharding.py
+++ b/tests/TestScripts/testSharding.py
@@ -25,7 +25,6 @@ def make_base_commandline(self_test_exe):
         "[generators]~[benchmarks]~[.]"
     ]
 
-
 def list_tests(self_test_exe):
     cmd = make_base_commandline(self_test_exe) + ['--list-tests']
 
@@ -81,30 +80,11 @@ def test_sharding(self_test_exe):
 
     check_listed_and_executed_tests_match(listed_tests, executed_tests)
 
-def test_invalid_shard_index(self_test_exe):
-    cmd = [
-        self_test_exe,
-        "--shard-count", "5",
-        "--shard-index", "5",
-        "[generators]~[benchmarks]~[.]"
-    ]
-
-    process = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    _, stderr = process.communicate()
-
-    errorString = str(stderr)
-
-    assert "The shard count (5) must be greater than the shard index (5)" in errorString, (
-        "Expected an error about the shard index but got {} instead".format(errorString)
-    )
-
 
 def main():
     self_test_exe, = sys.argv[1:]
 
     test_sharding(self_test_exe)
-    test_invalid_shard_index(self_test_exe)
 
 if __name__ == '__main__':
     sys.exit(main())