diff --git a/test/message/testcfg.py b/test/message/testcfg.py
index 7183ab333d8274..3c668459079f0a 100644
--- a/test/message/testcfg.py
+++ b/test/message/testcfg.py
@@ -127,8 +127,7 @@ def ListTests(self, current_path, path, arch, mode):
         file_path = file_prefix + ".js"
         output_path = file_prefix + ".out"
         if not exists(output_path):
-          print "Could not find %s" % output_path
-          continue
+          raise Exception("Could not find %s" % output_path)
         result.append(MessageTestCase(test, file_path, output_path,
                                       arch, mode, self.context, self))
     return result
diff --git a/test/pseudo-tty/testcfg.py b/test/pseudo-tty/testcfg.py
index 96b30253498857..40396db247e279 100644
--- a/test/pseudo-tty/testcfg.py
+++ b/test/pseudo-tty/testcfg.py
@@ -142,8 +142,7 @@ def ListTests(self, current_path, path, arch, mode):
         file_path = file_prefix + ".js"
         output_path = file_prefix + ".out"
         if not exists(output_path):
-          print "Could not find %s" % output_path
-          continue
+          raise Exception("Could not find %s" % output_path)
         result.append(TTYTestCase(test, file_path, output_path,
                                       arch, mode, self.context, self))
     return result