return non-zero exit code if no tests are collected

.travis.yml

@@ -28,7 +28,7 @@ env:
     - TESTENV=py35
     - TESTENV=pypy
 
-script: tox --recreate -i ALL=https://devpi.net/hpk/dev/ -e $TESTENV
+script: tox --recreate -e $TESTENV
 
 notifications:
   irc:

AUTHORS

@@ -15,6 +15,7 @@ Bob Ippolito
 Brian Dorsey
 Brian Okken
 Brianna Laugher
+Bruno Oliveira
 Carl Friedrich Bolz
 Charles Cloud
 Chris Lamb

CHANGELOG

@@ -58,6 +58,15 @@
 - Summary bar now is colored yellow for warning
   situations such as: all tests either were skipped or xpass/xfailed,
   or no tests were run at all (this is a partial fix for issue500).
+- fix issue812: pytest now exits with status code 5 in situations where no
+  tests were run at all, such as the directory given in the command line does
+  not contain any tests or as result of a command line option filters
+  all out all tests (-k for example).
+  Thanks Eric Siegerman (issue812) and Bruno Oliveira for the PR.
+
+- Summary bar now is colored yellow for warning
+  situations such as: all tests either were skipped or xpass/xfailed,
+  or no tests were run at all (related to issue500).
   Thanks Eric Siegerman.
 
 - New `testpaths` ini option: list of directories to search for tests

_pytest/main.py

@@ -19,6 +19,7 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 EXIT_USAGEERROR = 4
+EXIT_NOTESTSCOLLECTED = 5
 
 name_re = re.compile("^[a-zA-Z_]\w*$")
 
@@ -100,8 +101,10 @@ def wrap_session(config, doit):
             if excinfo.errisinstance(SystemExit):
                 sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
         else:
-            if session._testsfailed:
+            if session.testsfailed:
                 session.exitstatus = EXIT_TESTSFAILED
+            elif session.testscollected == 0:
+                session.exitstatus = EXIT_NOTESTSCOLLECTED
     finally:
         excinfo = None  # Explicitly break reference cycle.
         session.startdir.chdir()
@@ -509,7 +512,8 @@ def __init__(self, config):
         FSCollector.__init__(self, config.rootdir, parent=None,
                              config=config, session=self)
         self._fs2hookproxy = {}
-        self._testsfailed = 0
+        self.testsfailed = 0
+        self.testscollected = 0
         self.shouldstop = False
         self.trace = config.trace.root.get("collection")
         self._norecursepatterns = config.getini("norecursedirs")
@@ -527,11 +531,11 @@ def pytest_collectstart(self):
     @pytest.hookimpl(tryfirst=True)
     def pytest_runtest_logreport(self, report):
         if report.failed and not hasattr(report, 'wasxfail'):
-            self._testsfailed += 1
+            self.testsfailed += 1
             maxfail = self.config.getvalue("maxfail")
-            if maxfail and self._testsfailed >= maxfail:
+            if maxfail and self.testsfailed >= maxfail:
                 self.shouldstop = "stopping after %d failures" % (
-                    self._testsfailed)
+                    self.testsfailed)
     pytest_collectreport = pytest_runtest_logreport
 
     def isinitpath(self, path):
@@ -564,6 +568,7 @@ def perform_collect(self, args=None, genitems=True):
                 config=self.config, items=items)
         finally:
             hook.pytest_collection_finish(session=self)
+        self.testscollected = len(items)
         return items
 
     def _perform_collect(self, args, genitems):

_pytest/terminal.py

@@ -2,6 +2,8 @@
 
 This is a good source for looking at the various reporting hooks.
 """
+from _pytest.main import EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, \
+    EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED
 import pytest
 import pluggy
 import py
@@ -359,12 +361,15 @@ def pytest_sessionfinish(self, exitstatus):
         outcome = yield
         outcome.get_result()
         self._tw.line("")
-        if exitstatus in (0, 1, 2, 4):
+        summary_exit_codes = (
+            EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR,
+            EXIT_NOTESTSCOLLECTED)
+        if exitstatus in summary_exit_codes:
             self.summary_errors()
             self.summary_failures()
             self.summary_warnings()
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
-        if exitstatus == 2:
+        if exitstatus == EXIT_INTERRUPTED:
             self._report_keyboardinterrupt()
             del self._keyboardinterrupt_memo
         self.summary_deselected()

testing/acceptance_test.py

@@ -1,5 +1,7 @@
 import sys
 import py, pytest
+from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
+
 
 class TestGeneralUsage:
     def test_config_error(self, testdir):
@@ -147,7 +149,7 @@ def pytest_collect_directory():
                 pytest.skip("early")
         """)
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result.stdout.fnmatch_lines([
             "*1 skip*"
         ])
@@ -177,7 +179,7 @@ def test_issue93_initialnode_importing_capturing(self, testdir):
             sys.stderr.write("stder42\\n")
         """)
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         assert "should not be seen" not in result.stdout.str()
         assert "stderr42" not in result.stderr.str()
 
@@ -212,13 +214,13 @@ def test_issue109_sibling_conftests_not_loaded(self, testdir):
         sub2 = testdir.tmpdir.mkdir("sub2")
         sub1.join("conftest.py").write("assert 0")
         result = testdir.runpytest(sub2)
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         sub2.ensure("__init__.py")
         p = sub2.ensure("test_hello.py")
         result = testdir.runpytest(p)
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result = testdir.runpytest(sub1)
-        assert result.ret != 0
+        assert result.ret == EXIT_USAGEERROR
 
     def test_directory_skipped(self, testdir):
         testdir.makeconftest("""
@@ -228,7 +230,7 @@ def pytest_ignore_collect():
         """)
         testdir.makepyfile("def test_hello(): pass")
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result.stdout.fnmatch_lines([
             "*1 skipped*"
         ])
@@ -479,7 +481,7 @@ def test_invoke_with_string(self, capsys):
 
     def test_invoke_with_path(self, tmpdir, capsys):
         retcode = pytest.main(tmpdir)
-        assert not retcode
+        assert retcode == EXIT_NOTESTSCOLLECTED
         out, err = capsys.readouterr()
 
     def test_invoke_plugin_api(self, testdir, capsys):

testing/python/collect.py

@@ -1,6 +1,8 @@
 import sys
 from textwrap import dedent
 import pytest, py
+from _pytest.main import EXIT_NOTESTSCOLLECTED
+
 
 class TestModule:
     def test_failing_import(self, testdir):
@@ -906,7 +908,7 @@ class Test:
     """)
     result = testdir.runpytest()
     assert "TypeError" not in result.stdout.str()
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED    
 
 
 def test_collect_functools_partial(testdir):

testing/test_assertrewrite.py

@@ -12,6 +12,7 @@
 
 from _pytest.assertion import util
 from _pytest.assertion.rewrite import rewrite_asserts, PYTEST_TAG
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 
 
 def setup_module(mod):
@@ -429,7 +430,7 @@ def test_zipfile(self, testdir):
             import sys
             sys.path.append(%r)
             import test_gum.test_lizard""" % (z_fn,))
-        assert testdir.runpytest().ret == 0
+        assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
 
     def test_readonly(self, testdir):
         sub = testdir.mkdir("testing")
@@ -497,7 +498,7 @@ def test_package_without__init__py(self, testdir):
         pkg = testdir.mkdir('a_package_without_init_py')
         pkg.join('module.py').ensure()
         testdir.makepyfile("import a_package_without_init_py.module")
-        assert testdir.runpytest().ret == 0
+        assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
 
 class TestAssertionRewriteHookDetails(object):
     def test_loader_is_package_false_for_module(self, testdir):

testing/test_capture.py

@@ -10,6 +10,7 @@
 
 from _pytest import capture
 from _pytest.capture import CaptureManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 from py.builtin import print_
 
 needsosdup = pytest.mark.xfail("not hasattr(os, 'dup')")
@@ -365,7 +366,7 @@ def test_conftestlogging_is_shown(self, testdir):
         """)
         # make sure that logging is still captured in tests
         result = testdir.runpytest_subprocess("-s", "-p", "no:capturelog")
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result.stderr.fnmatch_lines([
             "WARNING*hello435*",
         ])

testing/test_collection.py

@@ -1,6 +1,6 @@
 import pytest, py
 
-from _pytest.main import Session
+from _pytest.main import Session, EXIT_NOTESTSCOLLECTED
 
 class TestCollector:
     def test_collect_versus_item(self):
@@ -247,10 +247,10 @@ def pytest_ignore_collect(path, config):
         p = testdir.makepyfile("def test_hello(): pass")
         result = testdir.runpytest(p)
         assert result.ret == 0
-        assert "1 passed" in result.stdout.str()
+        result.stdout.fnmatch_lines("*1 passed*")
         result = testdir.runpytest()
-        assert result.ret == 0
-        assert "1 passed" not in result.stdout.str()
+        assert result.ret == EXIT_NOTESTSCOLLECTED
+        result.stdout.fnmatch_lines("*collected 0 items*")
 
     def test_collectignore_exclude_on_option(self, testdir):
         testdir.makeconftest("""
@@ -264,7 +264,7 @@ def pytest_configure(config):
         testdir.mkdir("hello")
         testdir.makepyfile(test_world="def test_hello(): pass")
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         assert "passed" not in result.stdout.str()
         result = testdir.runpytest("--XX")
         assert result.ret == 0

testing/test_config.py

@@ -1,6 +1,7 @@
 import py, pytest
 
 from _pytest.config import getcfg, get_common_ancestor, determine_setup
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 
 class TestParseIni:
     def test_getcfg_and_config(self, testdir, tmpdir):
@@ -343,7 +344,7 @@ def test_invalid_options_show_extra_information(testdir):
 @pytest.mark.skipif("sys.platform == 'win32'")
 def test_toolongargs_issue224(testdir):
     result = testdir.runpytest("-m", "hello" * 500)
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
 
 def test_notify_exception(testdir, capfd):
     config = testdir.parseconfig()

testing/test_conftest.py

@@ -1,6 +1,7 @@
 from textwrap import dedent
 import py, pytest
 from _pytest.config import PytestPluginManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
 
 
 @pytest.fixture(scope="module", params=["global", "inpackage"])
@@ -166,7 +167,10 @@ def pytest_addoption(parser):
 def test_no_conftest(testdir):
     testdir.makeconftest("assert 0")
     result = testdir.runpytest("--noconftest")
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
+
+    result = testdir.runpytest()
+    assert result.ret == EXIT_USAGEERROR
 
 def test_conftest_existing_resultlog(testdir):
     x = testdir.mkdir("tests")

testing/test_helpconfig.py

@@ -1,3 +1,4 @@
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 import pytest
 
 def test_version(testdir, pytestconfig):
@@ -43,7 +44,7 @@ def pytest_hello(xyz):
             pass
     """)
     result = testdir.runpytest()
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
 
 def test_traceconfig(testdir):
     result = testdir.runpytest("--traceconfig")
@@ -54,14 +55,14 @@ def test_traceconfig(testdir):
 
 def test_debug(testdir, monkeypatch):
     result = testdir.runpytest_subprocess("--debug")
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
     p = testdir.tmpdir.join("pytestdebug.log")
     assert "pytest_sessionstart" in p.read()
 
 def test_PYTEST_DEBUG(testdir, monkeypatch):
     monkeypatch.setenv("PYTEST_DEBUG", "1")
     result = testdir.runpytest_subprocess()
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
     result.stderr.fnmatch_lines([
         "*pytest_plugin_registered*",
         "*manager*PluginManager*"

testing/test_junitxml.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from xml.dom import minidom
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 import py, sys, os
 from _pytest.junitxml import LogXML
 
@@ -298,7 +299,7 @@ def test_collect_error(self, testdir):
     def test_collect_skipped(self, testdir):
         testdir.makepyfile("import pytest; pytest.skip('xyz')")
         result, dom = runandparse(testdir)
-        assert not result.ret
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         node = dom.getElementsByTagName("testsuite")[0]
         assert_attr(node, skips=1, tests=0)
         tnode = node.getElementsByTagName("testcase")[0]

testing/test_pluginmanager.py

@@ -3,6 +3,7 @@
 import os
 
 from _pytest.config import get_config, PytestPluginManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 
 @pytest.fixture
 def pytestpm():
@@ -223,7 +224,7 @@ def test_plugin_skip(self, testdir, monkeypatch):
         p.copy(p.dirpath("skipping2.py"))
         monkeypatch.setenv("PYTEST_PLUGINS", "skipping2")
         result = testdir.runpytest("-rw", "-p", "skipping1", syspathinsert=True)
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result.stdout.fnmatch_lines([
             "WI1*skipped plugin*skipping1*hello*",
             "WI1*skipped plugin*skipping2*hello*",