Add option to disable error status code printing (#747)

PR #688 added functionality that broke the
[`JacobEvelyn/friends`](https://github.com/JacobEvelyn/friends)
build (see https://travis-ci.com/JacobEvelyn/friends/jobs/238680478),
which runs many CLI processes and parses the output. Some CLI
processes are expected to have non-success status codes, but
these tests also check that our CLI wrote the correct message
to STDERR, and with #688 the output to STDERR is now changed.

This commit adds a SimpleCov configuration flag to disable the
message added in #688.

CHANGELOG.md

@@ -4,6 +4,7 @@ Unrealeased
 ## Enhancements
 
 * If the minimum coverage is set to be greater than 100, a warning will be shown. See [#737](https://github.com/colszowka/simplecov/pull/737)
+* Add a configuration option to disable the printing of non-successful exit statuses. See [#747](https://github.com/colszowka/simplecov/pull/746) (thanks [@JacobEvelyn](https://github.com/JacobEvelyn))
 
 0.17.1 (2019-09-16)
 ===================

README.md

@@ -497,6 +497,21 @@ Then, SimpleCov will only run if you execute your tests like this:
 COVERAGE=true rake test
 ```
 
+## Errors and exit statuses
+
+To aid in debugging issues, if an error is raised, SimpleCov will print a message to `STDERR`
+with the exit status of the error, like:
+
+```
+SimpleCov failed with exit 1
+```
+
+This `STDERR` message can be disabled with:
+
+```
+SimpleCov.print_error_status = false
+```
+
 ## Profiles
 
 By default, SimpleCov's only config assumption is that you only want coverage reports for files inside your project

lib/simplecov.rb

@@ -186,7 +186,7 @@ def run_exit_tasks!
       # Force exit with stored status (see github issue #5)
       # unless it's nil or 0 (see github issue #281)
       if exit_status && exit_status.positive?
-        $stderr.printf("SimpleCov failed with exit %d\n", exit_status)
+        $stderr.printf("SimpleCov failed with exit %d\n", exit_status) if print_error_status
         Kernel.exit exit_status
       end
     end

lib/simplecov/configuration.rb

@@ -10,7 +10,7 @@
 #
 module SimpleCov
   module Configuration # rubocop:disable ModuleLength
-    attr_writer :filters, :groups, :formatter
+    attr_writer :filters, :groups, :formatter, :print_error_status
 
     #
     # The root for the project. This defaults to the
@@ -116,6 +116,14 @@ def formatters
       end
     end
 
+    #
+    # Whether we should print non-success status codes. This can be
+    # configured with the #print_error_status= method.
+    #
+    def print_error_status
+      defined?(@print_error_status) ? @print_error_status : true
+    end
+
     #
     # Certain code blocks (i.e. Ruby-implementation specific code) can be excluded from
     # the coverage metrics by wrapping it inside # :nocov: comment blocks. The nocov token

spec/configuration_spec.rb

@@ -10,6 +10,19 @@
   end
   let(:config) { config_class.new }
 
+  describe "#print_error_status" do
+    subject { config.print_error_status }
+
+    context "when not manually set" do
+      it { is_expected.to be true }
+    end
+
+    context "when manually set" do
+      before { config.print_error_status = false }
+      it { is_expected.to be false }
+    end
+  end
+
   describe "#tracked_files" do
     context "when configured" do
       let(:glob) { "{app,lib}/**/*.rb" }

spec/fixtures/frameworks/rspec_bad.rb

@@ -1,6 +1,9 @@
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "..", ".."))
 require "lib/simplecov"
 require "rspec"
+if ENV.key? "PRINT_ERROR_STATUS"
+  SimpleCov.print_error_status = ENV["PRINT_ERROR_STATUS"] == "true"
+end
 SimpleCov.start
 describe "exit status" do
   it "should exit with a non-zero exit status when assertion fails" do

spec/fixtures/frameworks/testunit_bad.rb

@@ -1,5 +1,8 @@
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "..", ".."))
 require "lib/simplecov"
+if ENV.key? "PRINT_ERROR_STATUS"
+  SimpleCov.print_error_status = ENV["PRINT_ERROR_STATUS"] == "true"
+end
 SimpleCov.start
 require "test/unit"
 class FooTest < Test::Unit::TestCase

spec/helper.rb

@@ -2,6 +2,7 @@
 
 require "rspec"
 require "stringio"
+require "open3"
 # loaded before simplecov to also capture parse time warnings
 require "support/fail_rspec_on_ruby_warning"
 require "simplecov"

spec/return_codes_spec.rb

@@ -13,24 +13,62 @@
       end
     end
 
-    it "has return code 0 when running testunit_good.rb" do
-      `ruby testunit_good.rb`
-      expect($?.exitstatus).to be_zero
+    before do
+      @stdout, @stderr, @status = Open3.capture3(command)
     end
 
-    it "has return code 0 when running rspec_good.rb" do
-      `rspec rspec_good.rb`
-      expect($?.exitstatus).to be_zero
+    shared_examples "good tests" do
+      it "has a zero exit status" do
+        expect(@status.exitstatus).to be_zero
+      end
+
+      it "prints nothing to STDERR" do
+        expect(@stderr).to be_empty
+      end
+    end
+
+    shared_examples "bad tests" do
+      context "with default configuration" do
+        it "has a non-zero exit status" do
+          expect(@status.exitstatus).not_to be_zero
+        end
+
+        it "prints a message to STDERR" do
+          expect(@stderr).to eq "SimpleCov failed with exit #{@status.exitstatus}\n"
+        end
+      end
+
+      context "when print_error_status is disabled" do
+        let(:command) { "PRINT_ERROR_STATUS=false " + super() }
+
+        it "has a non-zero exit status" do
+          expect(@status.exitstatus).not_to be_zero
+        end
+
+        it "does not print anything to STDERR" do
+          expect(@stderr).to be_empty
+        end
+      end
+    end
+
+    context "when running testunit_good.rb" do
+      let(:command) { "ruby testunit_good.rb" }
+      it_behaves_like "good tests"
+    end
+
+    context "when running rspec_good.rb" do
+      let(:command) { "rspec rspec_good.rb" }
+      it_behaves_like "good tests"
     end
 
-    it "has non-0 return code when running testunit_bad.rb" do
-      `ruby testunit_bad.rb`
-      expect($?.exitstatus).not_to be_zero
+    context "when running testunit_bad.rb" do
+      let(:command) { "ruby testunit_bad.rb" }
+      it_behaves_like "bad tests"
     end
 
-    it "has return code 1 when running rspec_bad.rb" do
-      `rspec rspec_bad.rb`
-      expect($?.exitstatus).not_to be_zero
+    context "when running rspec_bad.rb" do
+      let(:command) { "rspec rspec_bad.rb" }
+      it_behaves_like "bad tests"
     end
   end
 end