Randomize stdlib test order to uncover hidden assumptions

ruby · Oct 8, 2024 · 6690718 · 6690718
1 parent fc3c07f
commit 6690718
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
@@ -34,6 +34,8 @@ jobs:
             job: rubocop validate test_doc build test_generate_stdlib raap
           - ruby: "3.3"
             job: typecheck_test
+    env:
+      RANDOMIZE_STDLIB_TEST_ORDER: "true"
     steps:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1

diff --git a/Rakefile b/Rakefile
@@ -86,6 +86,11 @@ task :stdlib_test => :compile do
   test_files = FileList["test/stdlib/**/*_test.rb"].reject do |path|
     path =~ %r{Ractor} || path =~ %r{Encoding}
   end
+
+  if ENV["RANDOMIZE_STDLIB_TEST_ORDER"] == "true"
+    test_files.shuffle!
+  end
+
   sh "#{ruby} -Ilib #{bin}/test_runner.rb #{test_files.join(' ')}"
   # TODO: Ractor tests need to be run in a separate process
   sh "#{ruby} -Ilib #{bin}/test_runner.rb test/stdlib/Ractor_test.rb"