Implement bisection

Used to identify global state leak.

Co-Authored-By: Jean Boussier <[email protected]>

Author: etiennebarrie

TODO.md

@@ -1,8 +1,5 @@
 ### Wants
 
-- Test leak bisect
-  - See ci-queue bisect.
-
 - `-j` for forkless environments (Windows / JRuby / TruffleRuby)
 
 - `minitest/mocks`

fixtures/leak/leaky_test.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module TestedApp
+  singleton_class.attr_accessor :config
+  self.config = :default
+
+  class LeakyTest < Megatest::Test
+    test "leak cause" do
+      TestedApp.config = :leak
+      assert true
+    end
+
+    100.times do |i|
+      test "test something #{i}" do
+        assert true
+      end
+    end
+
+    test "leak sensitive" do
+      assert_equal :default, TestedApp.config
+    end
+  end
+end

lib/megatest/cli.rb

@@ -28,6 +28,7 @@ def run!
     undef_method :puts, :print # Should only use @out.puts or @err.puts
 
     RUNNERS = {
+      "bisect" => :bisect,
       "report" => :report,
       "run" => :run,
     }.freeze
@@ -51,6 +52,8 @@ def run
         report
       when nil, :run
         run_tests
+      when :bisect
+        bisect_tests
       else
         raise InvalidArgument, "Parsing failure"
       end
@@ -116,6 +119,32 @@ def report
       QueueReporter.new(@config, queue, @out).run(default_reporters) ? 0 : 1
     end
 
+    def bisect_tests
+      require "megatest/multi_process"
+
+      queue = @config.build_queue
+      raise InvalidArgument, "Distributed queues can't be bisected" if queue.distributed?
+
+      @config.selectors = Selector.parse(@argv)
+      Megatest.load_config(@config)
+      Megatest.init(@config)
+      test_cases = Megatest.load_tests(@config)
+      queue.populate(test_cases)
+      candidates = queue.dup
+
+      if test_cases.empty?
+        @err.puts "No tests to run"
+        return 1
+      end
+
+      unless failure = find_failing_test(queue)
+        @err.puts "No failing test"
+        return 1
+      end
+
+      bisect_queue(candidates, failure.test_id)
+    end
+
     private
 
     def default_reporters
@@ -141,6 +170,67 @@ def default_reporters
       reporters
     end
 
+    def find_failing_test(queue)
+      @config.max_consecutive_failures = 1
+      @config.jobs_count = 1
+
+      executor = MultiProcess::Executor.new(@config.dup, @out)
+      executor.run(queue, default_reporters)
+      queue.summary.failures.first
+    end
+
+    def bisect_queue(queue, failing_test_id)
+      err = Output.new(@err)
+      tests = queue.to_a
+      failing_test_index = tests.index { |test| test.id == failing_test_id }
+      failing_test = tests[failing_test_index]
+      suspects = tests[0...failing_test_index]
+
+      check_passing = @config.build_queue
+      check_passing.populate([failing_test])
+      executor = MultiProcess::Executor.new(@config.dup, @out, managed: true)
+      executor.run(check_passing, [])
+      unless check_passing.success?
+        err.puts err.red("Test failed by itself, no need to bisect")
+        return 1
+      end
+
+      run_index = 0
+      while suspects.size > 1
+        run_index += 1
+        err.puts "Attempt #{run_index}, #{suspects.size} suspects left."
+
+        before, after = suspects[0...(suspects.size / 2)], suspects[(suspects.size / 2)..]
+        candidates = @config.build_queue
+        candidates.populate(before + [failing_test])
+
+        executor = MultiProcess::Executor.new(@config.dup, @out, managed: true)
+        executor.run(candidates, default_reporters)
+
+        if candidates.success?
+          suspects = after
+        else
+          suspects = before
+        end
+
+        err.puts
+      end
+      suspect = suspects.first
+
+      validation_queue = @config.build_queue
+      validation_queue.populate([suspect, failing_test])
+      executor = MultiProcess::Executor.new(@config.dup, @out, managed: true)
+      executor.run(validation_queue, [])
+      if validation_queue.success?
+        err.puts err.red("Bisect inconclusive")
+        return 1
+      end
+
+      err.print "Found test leak: "
+      err.puts err.yellow "#{@config.program_name} #{Megatest.relative_path(suspect.location_id)} #{Megatest.relative_path(failing_test.location_id)}"
+      0
+    end
+
     def open_file(path)
       File.open(path, "w+")
     rescue Errno::ENOENT
@@ -176,6 +266,8 @@ def build_parser(runner)
           opts.banner = "Usage: #{@program_name} report [options]"
         when :run
           opts.banner = "Usage: #{@program_name} run [options] [files or directories]"
+        when :bisect
+          opts.banner = "Usage: #{@program_name} bisect [options] [files or directories]"
         else
           opts.banner = "Usage: #{@program_name} command [options] [files or directories]"
           opts.separator ""
@@ -190,6 +282,11 @@ def build_parser(runner)
           opts.separator "\treport\t\tWait for the queue to be entirely processed and report the status"
           opts.separator "\t\t\t  $ #{@program_name} report --queue redis://ci-queue.example.com --build-id $CI_BUILD_ID"
           opts.separator ""
+
+          opts.separator "\tbisect\t\tRepeatedly run subsets of the given tests."
+          opts.separator "\t\t\t  $ #{@program_name} bisect --seed 12345 test/integration"
+          opts.separator "\t\t\t  $ #{@program_name} bisect --queue path/to/test_order.log"
+          opts.separator ""
         end
 
         opts.separator ""
@@ -214,11 +311,13 @@ def build_parser(runner)
           @junit = path
         end
 
-        if runner == :run
+        if %i[run bisect].include?(runner)
           opts.on("--seed SEED", Integer, "The seed used to define run order") do |seed|
             @config.seed = seed
           end
+        end
 
+        if runner == :run
           opts.on("-j", "--jobs JOBS", Integer, "Number of processes to use") do |jobs|
             @config.jobs_count = jobs
           end
@@ -245,8 +344,10 @@ def build_parser(runner)
           @config.queue_url = queue_url
         end
 
-        opts.on("--build-id ID", String, "Unique identifier for the CI build") do |build_id|
-          @config.build_id = build_id
+        if %i[run report].include?(runner)
+          opts.on("--build-id ID", String, "Unique identifier for the CI build") do |build_id|
+            @config.build_id = build_id
+          end
         end
 
         if runner == :run

lib/megatest/config.rb

@@ -171,6 +171,11 @@ def initialize(env)
       CIService.configure(self, env)
     end
 
+    def initialize_dup(_)
+      super
+      @circuit_breaker = @circuit_breaker.dup
+    end
+
     def program_name
       @program_name || "megatest"
     end

lib/megatest/multi_process.rb

@@ -206,9 +206,10 @@ def tick
     class Executor
       attr_reader :wall_time
 
-      def initialize(config, out)
+      def initialize(config, out, managed: false)
         @config = config
         @out = Output.new(out, colors: config.colors)
+        @managed = managed
       end
 
       def concurrent?
@@ -260,7 +261,7 @@ def run(queue, reporters)
         @wall_time = Megatest.now - start_time
         reporters.each { |r| r.summary(self, queue, queue.summary) }
 
-        if @config.circuit_breaker.break?
+        if @config.circuit_breaker.break? && !@managed
           @out.error("Exited early because too many failures were encountered")
         end
 

lib/megatest/queue.rb

@@ -94,6 +94,11 @@ def initialize(results = [])
         @results = results
       end
 
+      def initialize_dup(_)
+        super
+        @results = @results.dup
+      end
+
       # When running distributed queues, it's possible
       # that a test is considered lost and end up with both
       # a successful and a failed result.
@@ -173,6 +178,18 @@ def initialize(config)
       @leases = {}
     end
 
+    def initialize_dup(_other)
+      super
+      @queue = @queue.dup
+      @summary = @summary.dup
+      @retries = @retries.dup
+      @leases = @leases.dup
+    end
+
+    def to_a
+      @queue.reverse
+    end
+
     def distributed?
       false
     end

test/megatest/selector_test.rb

@@ -50,6 +50,7 @@ def test_negative_path_loading
         "fixtures/generated_test.rb",
         "fixtures/inheritance/inheritance_test.rb",
         "fixtures/large/large_test.rb",
+        "fixtures/leak/leaky_test.rb",
         "fixtures/simple/error_test.rb",
         "fixtures/tags/tagged_test.rb",
       ]
@@ -59,13 +60,14 @@ def test_negative_path_loading
     def test_directory_path_shuffling
       selector = Selector.parse(["fixtures", "!", "fixtures/simple", "fixtures/simple/error_test.rb"])
       expected = [
-        "fixtures/errors/isolated_test.rb",
-        "fixtures/callbacks/callbacks_test.rb",
         "fixtures/simple/error_test.rb",
-        "fixtures/tags/tagged_test.rb",
+        "fixtures/leak/leaky_test.rb",
+        "fixtures/callbacks/callbacks_test.rb",
+        "fixtures/large/large_test.rb",
+        "fixtures/errors/isolated_test.rb",
         "fixtures/context/callbacks_test.rb",
         "fixtures/compat/compat_test.rb",
-        "fixtures/large/large_test.rb",
+        "fixtures/tags/tagged_test.rb",
         "fixtures/crash/crash_test.rb",
         "fixtures/inheritance/inheritance_test.rb",
         "fixtures/context/context_test.rb",