From 9bb122d9e6a94abcf33ec6342ddcea064822db9f Mon Sep 17 00:00:00 2001 From: Markus Schirp Date: Sun, 23 Apr 2023 23:19:01 +0000 Subject: [PATCH] Add test only runner --- Gemfile | 2 + Gemfile.lock | 10 +- docs/configuration.md | 1 + lib/mutant.rb | 3 + lib/mutant/bootstrap.rb | 28 +++- lib/mutant/cli/command/environment/test.rb | 24 +++- lib/mutant/integration.rb | 6 +- lib/mutant/integration/rspec.rb | 5 +- lib/mutant/reporter/cli.rb | 28 ++++ lib/mutant/reporter/cli/format.rb | 21 ++- lib/mutant/reporter/cli/printer/test.rb | 156 +++++++++++++++++++++ lib/mutant/result.rb | 82 +++++++++-- lib/mutant/test/runner.rb | 93 ++++++++++++ lib/mutant/test/runner/sink.rb | 46 ++++++ scripts/devloop.sh | 8 +- spec/support/corpus.rb | 2 +- spec/unit/mutant/bootstrap_spec.rb | 6 + spec/unit/mutant/cli_spec.rb | 5 + spec/unit/mutant/integration/rspec_spec.rb | 10 +- spec/unit/mutant/result_spec.rb | 8 -- 20 files changed, 502 insertions(+), 42 deletions(-) create mode 100644 lib/mutant/reporter/cli/printer/test.rb create mode 100644 lib/mutant/test/runner.rb create mode 100644 lib/mutant/test/runner/sink.rb diff --git a/Gemfile b/Gemfile index 00ec4f4e9..93dc5dd69 100644 --- a/Gemfile +++ b/Gemfile @@ -4,4 +4,6 @@ source 'https://rubygems.org' gemspec name: 'mutant' +gem 'rspec-core', path: '../rspec-core' + eval_gemfile 'Gemfile.shared' diff --git a/Gemfile.lock b/Gemfile.lock index 2c12782fe..4d81f6b9c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,9 @@ +PATH + remote: ../rspec-core + specs: + rspec-core (3.12.1) + rspec-support (~> 3.12.0) + PATH remote: . specs: @@ -33,8 +39,6 @@ GEM rspec-core (~> 3.12.0) rspec-expectations (~> 3.12.0) rspec-mocks (~> 3.12.0) - rspec-core (3.12.2) - rspec-support (~> 3.12.0) rspec-expectations (3.12.3) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) @@ -74,7 +78,7 @@ DEPENDENCIES mutant-license! parallel (~> 1.3) rspec (~> 3.10) - rspec-core (~> 3.10) + rspec-core! rspec-its (~> 1.3.0) rubocop (~> 1.7) diff --git a/docs/configuration.md b/docs/configuration.md index ed7b5b4bd..7292c821e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -89,6 +89,7 @@ integration: # Below shows an example configuring rspec to use a static seed from the config file. arguments: - --fail-fast # rspec integration default, keep this when specifying manual options! + - --force-color # rspec integration default, keep this if colored output is not an issue for you - --seed # option - '0' # option value, needs to be a string. - spec # rspec integration default, tell rspec integration where to find specs diff --git a/lib/mutant.rb b/lib/mutant.rb index ac042f8dd..46dc7c70d 100644 --- a/lib/mutant.rb +++ b/lib/mutant.rb @@ -204,6 +204,8 @@ module Mutant require 'mutant/expression/namespace' require 'mutant/expression/parser' require 'mutant/test' + require 'mutant/test/runner' + require 'mutant/test/runner/sink' require 'mutant/timer' require 'mutant/integration' require 'mutant/integration/null' @@ -243,6 +245,7 @@ module Mutant require 'mutant/reporter/cli/printer/mutation_result' require 'mutant/reporter/cli/printer/status_progressive' require 'mutant/reporter/cli/printer/subject_result' + require 'mutant/reporter/cli/printer/test' require 'mutant/reporter/cli/format' require 'mutant/repository' require 'mutant/repository/diff' diff --git a/lib/mutant/bootstrap.rb b/lib/mutant/bootstrap.rb index 7b6f3bc77..fb9d38407 100644 --- a/lib/mutant/bootstrap.rb +++ b/lib/mutant/bootstrap.rb @@ -29,8 +29,6 @@ module Bootstrap # @param [Env] env # # @return [Either] - # - # rubocop:disable Metrics/MethodLength def self.call(env) env.record(:bootstrap) do env = load_hooks(env) @@ -47,6 +45,31 @@ def self.call(env) selected_subjects.flat_map(&:mutations) end + setup_integration( + env: env, + mutations: mutations, + selected_subjects: selected_subjects + ) + end + end + + # Run test only bootstrap + # + # @param [Env] env + # + # @return [Either] + def self.call_test(env) + env.record(:bootstrap) do + setup_integration( + env: load_hooks(env), + mutations: [], + selected_subjects: [] + ) + end + end + + def self.setup_integration(env:, mutations:, selected_subjects:) + env.record(__method__) do Integration.setup(env).fmap do |integration| env.with( integration: integration, @@ -57,7 +80,6 @@ def self.call(env) end end end - # rubocop:enable Metrics/MethodLength def self.load_hooks(env) env.record(__method__) do diff --git a/lib/mutant/cli/command/environment/test.rb b/lib/mutant/cli/command/environment/test.rb index ffdb8a8b2..604a38cdd 100644 --- a/lib/mutant/cli/command/environment/test.rb +++ b/lib/mutant/cli/command/environment/test.rb @@ -8,6 +8,16 @@ class Test < self NAME = 'test' SHORT_DESCRIPTION = 'test subcommands' + private + + def bootstrap + env = Env.empty(world, @config) + + env + .record(:config) { Config.load(cli_config: @config, world: world) } + .bind { |config| Bootstrap.call_test(env.with(config: config)) } + end + class List < self NAME = 'list' SHORT_DESCRIPTION = 'List tests detected in the environment' @@ -28,7 +38,19 @@ def list_tests(env) end end - SUBCOMMANDS = [List].freeze + class Run < self + NAME = 'run' + SHORT_DESCRIPTION = 'Run tests' + SUBCOMMANDS = EMPTY_ARRAY + + private + + def action + bootstrap.bind(&Mutant::Test::Runner.method(:call)) + end + end + + SUBCOMMANDS = [List, Run].freeze end # Test end # Environment end # Command diff --git a/lib/mutant/integration.rb b/lib/mutant/integration.rb index bf2a6eb68..28905b375 100644 --- a/lib/mutant/integration.rb +++ b/lib/mutant/integration.rb @@ -4,7 +4,11 @@ module Mutant # Abstract base class mutant test framework integrations class Integration - include AbstractType, Adamantium, Anima.new(:arguments, :expression_parser, :world) + include AbstractType, Adamantium, Anima.new( + :arguments, + :expression_parser, + :world + ) LOAD_MESSAGE = <<~'MESSAGE' Unable to load integration mutant-%s: diff --git a/lib/mutant/integration/rspec.rb b/lib/mutant/integration/rspec.rb index 69ff4652b..3ba7094eb 100644 --- a/lib/mutant/integration/rspec.rb +++ b/lib/mutant/integration/rspec.rb @@ -24,7 +24,7 @@ class Rspec < self ALL_EXPRESSION = Expression::Namespace::Recursive.new(scope_name: nil) EXPRESSION_CANDIDATE = /\A([^ ]+)(?: )?/.freeze EXIT_SUCCESS = 0 - DEFAULT_CLI_OPTIONS = %w[--fail-fast spec].freeze + DEFAULT_CLI_OPTIONS = %w[--fail-fast --force-color spec].freeze TEST_ID_FORMAT = 'rspec:%d:%s/%s' private_constant(*constants(false)) @@ -42,7 +42,7 @@ def initialize(*) # # @return [self] def setup - @runner.setup($stderr, $stdout) + @runner.setup(world.stderr, world.stdout) example_group_map reset_examples self @@ -55,6 +55,7 @@ def setup # # @return [Result::Test] def call(tests) + reset_examples setup_examples(tests.map(&all_tests_index)) start = timer.now passed = @runner.run_specs(@rspec_world.ordered_example_groups).equal?(EXIT_SUCCESS) diff --git a/lib/mutant/reporter/cli.rb b/lib/mutant/reporter/cli.rb index d8e52bc7e..8a8281003 100644 --- a/lib/mutant/reporter/cli.rb +++ b/lib/mutant/reporter/cli.rb @@ -28,6 +28,16 @@ def start(env) self end + # Report test start + # + # @param [Env] env + # + # @return [self] + def test_start(env) + write(format.test_start(env)) + self + end + # Report progress object # # @param [Parallel::Status] status @@ -38,6 +48,14 @@ def progress(status) self end + # Report progress object + # + # @return [self] + def test_progress(status) + write(format.test_progress(status)) + self + end + # Report delay in seconds # # @return [Float] @@ -65,6 +83,16 @@ def report(env) self end + # Report env + # + # @param [Result::Env] env + # + # @return [self] + def test_report(env) + Printer::Test::EnvResult.call(output: output, object: env) + self + end + private def write(frame) diff --git a/lib/mutant/reporter/cli/format.rb b/lib/mutant/reporter/cli/format.rb index b92e7c4d5..ecdfd8344 100644 --- a/lib/mutant/reporter/cli/format.rb +++ b/lib/mutant/reporter/cli/format.rb @@ -18,11 +18,14 @@ class Format # Progress representation # - # @param [Runner::Status] status - # # @return [String] abstract_method :progress + # Progress representation + # + # @return [String] + abstract_method :test_progress + # Report delay in seconds # # @return [Float] @@ -69,6 +72,13 @@ def start(env) format(Printer::Env, env) end + # Test start representation + # + # @return [String] + def test_start(env) + format(Printer::Test::Env, env) + end + # Progress representation # # @return [String] @@ -76,6 +86,13 @@ def progress(status) format(Printer::StatusProgressive, status) end + # Progress representation + # + # @return [String] + def test_progress(status) + format(Printer::Test::StatusProgressive, status) + end + private def new_buffer diff --git a/lib/mutant/reporter/cli/printer/test.rb b/lib/mutant/reporter/cli/printer/test.rb new file mode 100644 index 000000000..18bf5b645 --- /dev/null +++ b/lib/mutant/reporter/cli/printer/test.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +module Mutant + class Reporter + class CLI + class Printer + class Test < self + # Printer for test config + class Config < self + + # Report configuration + # + # @param [Mutant::Config] config + # + # @return [undefined] + # + def run + info('Fail-Fast: %s', object.fail_fast) + info('Integration: %s', object.integration.name || 'null') + info('Jobs: %s', object.jobs || 'auto') + end + end # Config + + # Env printer + class Env < self + delegate( + :amount_available_tests, + :amount_mutations, + :amount_selected_tests, + :amount_subjects, + :amount_all_tests, + :config, + :test_subject_ratio + ) + + FORMATS = [ + [:info, 'Tests: %s', :amount_all_tests] + ].each(&:freeze) + + # Run printer + # + # @return [undefined] + def run + info('Test environment:') + visit(Config, config) + FORMATS.each do |report, format, value| + __send__(report, format, __send__(value)) + end + end + end # Env + + class EnvProgress < self + delegate( + :amount_test_results, + :amount_tests_failed, + :amount_tests_success, + :env, + :overhead, + :runtime, + :testtime + ) + + FORMATS = [ + [:info, 'Test-Results: %0d', :amount_test_results ], + [:info, 'Test-Failed: %0d', :amount_tests_failed ], + [:info, 'Test-Success: %0d', :amount_tests_success ], + [:info, 'Runtime: %0.2fs', :runtime ], + [:info, 'Testtime: %0.2fs', :testtime ], + [:info, 'Overhead: %0.2f%%', :overhead_percent ] + ].each(&:freeze) + + # Run printer + # + # @return [undefined] + def run + visit(Env, object.env) + FORMATS.each do |report, format, value| + __send__(report, format, __send__(value)) + end + end + + private + + def overhead_percent + (overhead / runtime) * 100 + end + end # EnvProgress + + # Full env result reporter + class EnvResult < self + + # Run printer + # + # @return [undefined] + def run + visit_collection(Result, object.failed_test_results) + visit(EnvProgress, object) + end + + end # EnvResult + + # Reporter for mutation results + class Result < self + + # Run report printer + # + # @return [undefined] + def run + puts(object.output) + end + + end # Result + + # Reporter for progressive output format on scheduler Status objects + class StatusProgressive < self + FORMAT = 'progress: %02d/%02d failed: %d runtime: %0.02fs testtime: %0.02fs tests/s: %0.02f' + + delegate( + :amount_test_results, + :amount_tests, + :amount_tests_failed, + :testtime, + :overhead, + :runtime + ) + + # Run printer + # + # @return [undefined] + def run + status( + FORMAT, + amount_test_results, + amount_tests, + amount_tests_failed, + runtime, + testtime, + tests_per_second + ) + end + + private + + def object + super().payload + end + + def tests_per_second + amount_test_results / runtime + end + end # StatusProgressive + end # Test + end # Printer + end # CLI + end # Reporter +end # Mutant diff --git a/lib/mutant/result.rb b/lib/mutant/result.rb index f701210fa..0103f8882 100644 --- a/lib/mutant/result.rb +++ b/lib/mutant/result.rb @@ -50,17 +50,6 @@ def delegate(name, target) private_constant(*constants(false)) - # Mutant overhead running mutatet tests - # - # This is NOT the overhead of mutation testing, just an engine specific - # measurement for the efficiency of the parallelization engine, kill - # isolation etc. - # - # @return [Float] - def overhead - runtime - killtime - end - # Hook called when module gets included # # @param [Class, Module] host @@ -102,6 +91,10 @@ def failed_subject_results sum :amount_timeouts, :subject_results sum :killtime, :subject_results + def overhead + runtime - killtime + end + # Amount of mutations # # @return [Integer] @@ -116,13 +109,71 @@ def amount_mutations def stop? env.config.fail_fast && !subject_results.all?(&:success?) end - end # Env + # TestEnv result object + class TestEnv + include Result, Anima.new( + :env, + :runtime, + :test_results + ) + + def overhead + runtime - testtime + end + + def inspect + '' + end + + # Test if run is successful + # + # @return [Boolean] + def success? + amount_tests_failed.equal?(0) + end + memoize :success? + + # Failed subject results + # + # @return [Array] + def failed_test_results + test_results.reject(&:success?) + end + memoize :failed_test_results + + def stop? + env.config.fail_fast && !test_results.all?(&:success?) + end + + def testtime + test_results.map(&:runtime).reduce(0, :+) + end + + def amount_tests + env.integration.all_tests.length + end + + def amount_test_results + test_results.length + end + + def amount_tests_failed + failed_test_results.length + end + + def amount_tests_success + test_results.count(&:passed) + end + end # TestEnv + # Test result class Test include Anima.new(:passed, :runtime) + alias_method :success?, :passed + class VoidValue < self include Singleton @@ -138,6 +189,13 @@ def initialize end # VoidValue end # Test + # Test result + class Test2 + include Anima.new(:passed, :runtime, :output) + + alias_method :success?, :passed + end # Test2 + # Subject result class Subject include CoverageMetric, Result, Anima.new( diff --git a/lib/mutant/test/runner.rb b/lib/mutant/test/runner.rb new file mode 100644 index 000000000..c9eee3e15 --- /dev/null +++ b/lib/mutant/test/runner.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Mutant + class Test + module Runner + # Run against env + # + # @return [Either] + def self.call(env) + reporter(env).test_start(env) + + Either::Right.new(run_tests(env)) + end + + def self.run_tests(env) + reporter = reporter(env) + + env + .record(:tests) { run_driver(reporter, async_driver(env)) } + .tap { |result| env.record(:report) { reporter.test_report(result) } } + end + private_class_method :run_tests + + def self.async_driver(env) + Parallel.async(env.world, test_config(env)) + end + private_class_method :async_driver + + def self.run_driver(reporter, driver) + Signal.trap('INT') do + driver.stop + end + + loop do + status = driver.wait_timeout(reporter.delay) + break status.payload if status.done? + reporter.test_progress(status) + end + end + private_class_method :run_driver + + def self.test_config(env) + Parallel::Config.new( + block: ->(index) { run_test_index(env, index) }, + jobs: env.config.jobs, + on_process_start: ->(index) {}, + process_name: 'mutant-test-runner-process', + sink: Sink.new(env), + source: Parallel::Source::Array.new(jobs: env.integration.all_tests.each_index.to_a.reverse), + thread_name: 'mutant-test-runner-thread' + ) + end + private_class_method :test_config + + def self.run_test_index(env, test_index) + test_result = nil + + output = with_capture(env.world) do + test_result = env.integration.call([env.integration.all_tests.fetch(test_index)]) + end + + Result::Test2.new( + passed: test_result.passed, + runtime: test_result.runtime, + output: output + ) + end + private_class_method :run_test_index + + def self.with_capture(world) + stderr, stdout = world.stderr, world.stdout + original_stdout = stderr.dup + original_stderr = stderr.dup + Tempfile.create('capture') do |io| + stdout.reopen(io) + stdout.sync = true + stderr.reopen(stdout) + yield + io.rewind + io.read + end + ensure + stdout.reopen(original_stdout) + stderr.reopen(original_stderr) + end + + def self.reporter(env) + env.config.reporter + end + private_class_method :reporter + end # Runner + end # Test +end # Mutant diff --git a/lib/mutant/test/runner/sink.rb b/lib/mutant/test/runner/sink.rb new file mode 100644 index 000000000..dd37d9cbb --- /dev/null +++ b/lib/mutant/test/runner/sink.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Mutant + class Test + module Runner + class Sink + include Concord.new(:env) + + # Initialize object + # + # @return [undefined] + def initialize(*) + super + @start = env.world.timer.now + @test_results = [] + end + + # Runner status + # + # @return [Result::Env] + def status + Result::TestEnv.new( + env: env, + runtime: env.world.timer.now - @start, + test_results: @test_results + ) + end + + # Test if scheduling stopped + # + # @return [Boolean] + def stop? + status.stop? + end + + # Handle mutation finish + # + # @return [self] + def result(result) + @test_results << result + self + end + end # Sink + end # Runner + end # Test +end # Mutant diff --git a/scripts/devloop.sh b/scripts/devloop.sh index e17dfda4c..adb53ad7c 100755 --- a/scripts/devloop.sh +++ b/scripts/devloop.sh @@ -1,5 +1,5 @@ -while inotifywait lib/**/*.rb meta/**/*.rb spec/**/*.rb Gemfile Gemfile.shared mutant.gemspec; do - bundle exec rspec spec/unit -fd --fail-fast --order default \ - && bundle exec mutant run --since main --fail-fast --zombie -- 'Mutant*' \ - && bundle exec rubocop +while inotifywait ../unparser/**/*.rb {lib,spec}/**/*.rb Gemfile Gemfile.shared mutant.gemspec; do + bundle exec mutant environment test run --fail-fast # +## && bundle exec mutant run --since main --fail-fast --zombie -- 'Mutant*' \ +## && bundle exec rubocop done diff --git a/spec/support/corpus.rb b/spec/support/corpus.rb index 6c88dd3b9..8743574ce 100644 --- a/spec/support/corpus.rb +++ b/spec/support/corpus.rb @@ -210,7 +210,7 @@ def effective_ruby_paths # # @return [Pathname] def repo_path - TMP.join(name) + TMP.join("#{name}-#{Digest::SHA256.hexdigest(inspect)}") end # Test if installation should be skipped diff --git a/spec/unit/mutant/bootstrap_spec.rb b/spec/unit/mutant/bootstrap_spec.rb index 729c0b960..0d930e21c 100644 --- a/spec/unit/mutant/bootstrap_spec.rb +++ b/spec/unit/mutant/bootstrap_spec.rb @@ -179,6 +179,12 @@ def require(_); end arguments: [:mutation_generate], reaction: { yields: [] } }, + { + receiver: world, + selector: :record, + arguments: [:setup_integration], + reaction: { yields: [] } + }, { receiver: Mutant::Integration, selector: :setup, diff --git a/spec/unit/mutant/cli_spec.rb b/spec/unit/mutant/cli_spec.rb index 88b11c673..9064c6c0b 100644 --- a/spec/unit/mutant/cli_spec.rb +++ b/spec/unit/mutant/cli_spec.rb @@ -730,6 +730,11 @@ def self.main_body bootstrap_result end + allow(Mutant::Bootstrap).to receive(:call_test) do |env| + events << [:bootstrap, env.inspect] + bootstrap_result + end + allow(Mutant::Mutation::Runner).to receive(:call) do |env| events << [:runner, env.inspect] runner_result diff --git a/spec/unit/mutant/integration/rspec_spec.rb b/spec/unit/mutant/integration/rspec_spec.rb index 444c9d8c1..ed73c681e 100644 --- a/spec/unit/mutant/integration/rspec_spec.rb +++ b/spec/unit/mutant/integration/rspec_spec.rb @@ -11,7 +11,7 @@ ) end - let(:expected_rspec_cli) { %w[--fail-fast spec] } + let(:expected_rspec_cli) { %w[--fail-fast --force-color spec] } let(:integration_arguments) { [] } let(:rspec_options) { instance_double(RSpec::Core::ConfigurationOptions) } let(:rspec_runner) { instance_double(RSpec::Core::Runner) } @@ -191,8 +191,8 @@ before do expect(rspec_runner).to receive(:setup) do |error, stdout| - expect(error).to be($stderr) - expect(stdout).to be($stdout) + expect(error).to be(world.stderr) + expect(stdout).to be(world.stdout) end end @@ -206,8 +206,8 @@ before do expect(rspec_runner).to receive(:setup) do |error, stdout| - expect(error).to be($stderr) - expect(stdout).to be($stdout) + expect(error).to be(world.stderr) + expect(stdout).to be(world.stdout) end allow(rspec_runner).to receive_messages(run_specs: exit_status) end diff --git a/spec/unit/mutant/result_spec.rb b/spec/unit/mutant/result_spec.rb index 4039711f3..a1b6f8eab 100644 --- a/spec/unit/mutant/result_spec.rb +++ b/spec/unit/mutant/result_spec.rb @@ -22,12 +22,4 @@ def collection expect(object.length).to be(1) end end - - describe '#overhead' do - subject { object.overhead } - - it 'returns difference between runtime and killtime' do - should eql(2.0) - end - end end