Provide the possibility of a callback to do something when a flaky spec is deteced

Author: Federico Aldunate

README.md

@@ -51,6 +51,11 @@ RSpec.configure do |config|
     ex.run_with_retry retry: 3
   end
 
+  # callback to be run when a flaky test is detected
+  config.flaky_test_callback = proc do |example|
+    Rspec::Watchdog::Reporter.report(example)
+  end
+
   # callback to be run between retries
   config.retry_callback = proc do |ex|
     # run some additional clean up task - can be filtered by example metadata
@@ -90,7 +95,7 @@ You can call `ex.run_with_retry(opts)` on an individual example.
 - __:exceptions_to_hard_fail__(default: *[]*) List of exceptions that will trigger an immediate test failure without retry. Takes precedence over __:exceptions_to_retry__
 - __:exceptions_to_retry__(default: *[]*) List of exceptions that will trigger a retry (when empty, all exceptions will)
 - __:retry_callback__(default: *nil*) Callback function to be called between retries
-
+- __:flaky_test_callback__(default: *nil*) Callback function to be called when a flaky test is detected (when a test fails but then passes on a subsequent attempt)p
 
 ## Environment Variables
 - __RSPEC_RETRY_RETRY_COUNT__ can override the retry counts even if a retry count is set in an example or default_retry_count is set in a configuration.

lib/rspec/rebound.rb

@@ -33,6 +33,9 @@ def self.setup
         # Callback between retries
         config.add_setting :retry_callback, :default => nil
 
+        # Callback for flaky tests
+        config.add_setting :flaky_test_callback, :default => nil
+
         config.around(:each) do |ex|
           ex.run_with_retry
         end
@@ -122,6 +125,14 @@ def run
         example.clear_exception
         ex.run
 
+        if example.exception.nil?
+          # If it's a flaky test, call the callback
+          if attempts > 0 && RSpec.configuration.flaky_test_callback
+
+            example.example_group_instance.instance_exec(example, &RSpec.configuration.flaky_test_callback)
+          end
+        end
+
         self.attempts += 1
 
         break if example.exception.nil?

rspec-rebound.gemspec

@@ -19,4 +19,5 @@ Gem::Specification.new do |gem|
   gem.version       = RSpec::Rebound::VERSION
   gem.add_runtime_dependency "rspec-core", "~> 3.3"
   gem.add_development_dependency "rspec", "~> 3.3"
+  gem.add_development_dependency "debug", "~> 1.0"
 end

spec/spec_helper.rb

@@ -2,10 +2,8 @@
 require 'rspec/core/sandbox'
 
 require 'rspec/rebound'
-if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2')
-  require "pry-debugger"
-else
-  require "pry-byebug"
+if Gem::Version.new(RUBY_VERSION) > Gem::Version.new('3')
+  require 'debug'
 end
 
 RSpec.configure do |config|