diff --git a/.github/workflows/rspec-distrib-tests.yml b/.github/workflows/rspec-distrib-tests.yml index 5129871..80c2ea6 100644 --- a/.github/workflows/rspec-distrib-tests.yml +++ b/.github/workflows/rspec-distrib-tests.yml @@ -21,9 +21,9 @@ jobs: working-directory: rspec-distrib - name: Run specs - run: bundle exec rspec spec/ + run: bundle exec rspec spec working-directory: rspec-distrib - name: Run features - run: bundle exec rspec features/ + run: bundle exec rspec features working-directory: rspec-distrib diff --git a/README.md b/README.md index 4e3d762..1b36d57 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,24 @@ ## What's *-distrib? -This is a collection of gems for running test in parallel on +This is a collection of gems for running tests in parallel on multiple machines/processes. -* [distrib-core](./distrib-core/README.md) -* [features-parser](./features-parser/README.md) -* [rspec-distrib](./rspec-distrib/README.md) +This is a monorepo. Please follow to each gem's directory to get more insights. + +## distrib-core + +[distrib-core](./distrib-core/README.md) gem defines core classes. It can be used as a building block for various *-distrib runners. + +## rspec-distrib + +[rspec-distrib](./rspec-distrib/README.md) gem is an implementation of distrib runner for [RSpec](https://rspec.info/) tests. + +## cucumber-distrib + +cucumber-distrib gem is an implementation of distrib runner for [Cucumber](https://cucumber.io/). +It will be open-sourced down the road. Stay tuned! + +## features-parser + +[features-parser](./features-parser/README.md) gem is a helper for cucumber-distrib runner. diff --git a/distrib-core/.rubocop.yml b/distrib-core/.rubocop.yml index 7cc356b..2875baa 100644 --- a/distrib-core/.rubocop.yml +++ b/distrib-core/.rubocop.yml @@ -4,6 +4,7 @@ require: AllCops: DisplayCopNames: true NewCops: enable + SuggestExtensions: false Exclude: - coverage/**/* - bundle/**/* diff --git a/distrib-core/Gemfile.lock b/distrib-core/Gemfile.lock index c03468b..e002cc2 100644 --- a/distrib-core/Gemfile.lock +++ b/distrib-core/Gemfile.lock @@ -66,7 +66,7 @@ GEM simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) strscan (3.1.0) - timecop (0.9.9) + timecop (0.9.10) unicode-display_width (2.5.0) yard (0.9.36) diff --git a/distrib-core/README.md b/distrib-core/README.md index 47909c8..8cf67cf 100644 --- a/distrib-core/README.md +++ b/distrib-core/README.md @@ -1,14 +1,16 @@ # distrib-core -Is a common core module for [rspec-distrib](../rspec-distrib). +This is a common core module for *-distrib runners. +At this point for [rspec-distrib](../rspec-distrib). ## Installation Add the gem to the application's Gemfile: ```ruby -gem 'distrib-core', git: 'git@github.com:toptal/test-distrib.git', - glob: 'distrib-core/*.gemspec' +git 'git@github.com:toptal/test-distrib.git', branch: 'main' do + gem 'distrib-core', require: false, group: [:test] +end ``` ## Getting started diff --git a/distrib-core/lib/distrib-core.rb b/distrib-core/lib/distrib-core.rb index 3402411..f42f845 100644 --- a/distrib-core/lib/distrib-core.rb +++ b/distrib-core/lib/distrib-core.rb @@ -1,2 +1,17 @@ -# So we can require it as 'distrib-core'. -require 'distrib_core' +require 'distrib_core/core_ext/drb_tcp_socket' +require 'distrib_core/logger_broadcaster' +require 'distrib_core/leader' +require 'distrib_core/configuration' +require 'distrib_core/distrib' +require 'distrib_core/drb_helper' +require 'distrib_core/metrics' +require 'distrib_core/received_signals' +require 'distrib_core/worker' + +# A core module. Has a quick alias to configuration. +module DistribCore + # Alias to {DistribCore::Configuration.current} + def self.configuration + Configuration.current + end +end diff --git a/distrib-core/lib/distrib_core.rb b/distrib-core/lib/distrib_core.rb deleted file mode 100644 index f42f845..0000000 --- a/distrib-core/lib/distrib_core.rb +++ /dev/null @@ -1,17 +0,0 @@ -require 'distrib_core/core_ext/drb_tcp_socket' -require 'distrib_core/logger_broadcaster' -require 'distrib_core/leader' -require 'distrib_core/configuration' -require 'distrib_core/distrib' -require 'distrib_core/drb_helper' -require 'distrib_core/metrics' -require 'distrib_core/received_signals' -require 'distrib_core/worker' - -# A core module. Has a quick alias to configuration. -module DistribCore - # Alias to {DistribCore::Configuration.current} - def self.configuration - Configuration.current - end -end diff --git a/distrib-core/lib/distrib_core/configuration.rb b/distrib-core/lib/distrib_core/configuration.rb index 6137fcb..c4db9dc 100644 --- a/distrib-core/lib/distrib_core/configuration.rb +++ b/distrib-core/lib/distrib_core/configuration.rb @@ -4,12 +4,12 @@ require 'distrib_core/logger_broadcaster' module DistribCore - # This module contains shared attrs instantiated by specific configuration classes. + # This module contains shared attributes instantiated by specific configuration classes. # # @see DistribCore::Distrib#configure module Configuration class << self - # Set global configuration. Can be set only one time + # Set global configuration. Can be set only once. # # @param configuration [DistribCore::Configuration] def current=(configuration) @@ -26,6 +26,8 @@ def current TIMEOUT_STRATEGIES = %i[repush release].freeze + # A test provider. It should be a callable object that returns a list of tests to execute. + # # @example Override default list of the tests: # ...configure do |config| # config.tests_provider = -> { @@ -34,6 +36,9 @@ def current # end attr_writer :tests_provider + # An object to handle errors. Defines strategy of managing retries of tests. + # It should be an instance of {DistribCore::Leader::ErrorHandler}. + # # @example Specify object to process exceptions during execution # ...configure do |config| # config.error_handler = MyErrorHandler.new @@ -42,6 +47,8 @@ def current attr_writer :logger + # Set timeout for tests. It can be a number of seconds or a callable object that returns a number of seconds. + # # @example Set equal timeout for all tests to 30 seconds: # ...configure do |config| # config.test_timeout = 30 # seconds @@ -55,19 +62,27 @@ def current # end attr_accessor :test_timeout - # @example Set how long leader will wait before first test processed by workers. Leader will exit if no tests picked in this period + # Specify how long leader will wait before first test processed by workers. Leader will exit if no tests picked in this period. + # Value is in seconds. + # + # @example Set timeout to 10 minutes # ...configure do |config| # config.first_test_picked_timeout = 10*60 # 10 minutes # end attr_accessor :first_test_picked_timeout - # @example Specify custom options for DRb service. Defaults are `{ safe_level: 1 }`. @see `DRb::DRbServer.new` for complete list + # Specify custom options for DRb service. Defaults are `{ safe_level: 1 }`. + # @see `DRb::DRbServer.new` for complete list. + # + # @example # ...configure do |config| # config.drb = {safe_level: 0, verbose: true} # end attr_accessor :drb - # @example Specify custom block to pre-process examples before reporting them to the leader. Useful to add additional information about workers. + # Specify custom block to pre-process examples before reporting them to the leader. Useful to add additional information about workers. + # + # @example # ...configure do |config| # config.before_test_report = -> (file_name, example_groups) do # example_groups.each { |eg| eg.metadata[:custom] = 'foo' } @@ -75,7 +90,9 @@ def current # end attr_accessor :before_test_report - # @example Specify custom block which will be called on leader after run. + # Specify custom block which will be called on leader after run. + # + # @example # ...configure do |config| # config.on_finish = -> () do # 'Whatever logic before leader exit' @@ -83,13 +100,33 @@ def current # end attr_accessor :on_finish + # Specify a debug logger. + # # @example Disable (mute) debug logger # ...configure do |config| # config.debug_logger = Logger.new(nil) # end attr_writer :debug_logger - attr_accessor :tests_processing_stopped_timeout, :drb_tcp_socket_connection_timeout, :leader_connection_attempts + # Specify how long leader will wait for test to be picked by a worker. Leader will exit if no test is picked in this period. + # Value is in seconds. + attr_accessor :tests_processing_stopped_timeout + + # Specify a connection timeout on DRb Socket. + # Value is in seconds. + # @see {DRb::DRbTCPSocket} monkey patch. + attr_accessor :drb_tcp_socket_connection_timeout + + # Specify how many times worker will try to connect to the leader. + attr_accessor :leader_connection_attempts + + # Specify a strategy for handling timed out tests. + # Can be one of `:repush` or `:release`. + # + # @example Change strategy to `:release` + # ...configure do |config| + # config.timeout_strategy = :release + # end attr_reader :timeout_strategy # Initialize configuration with default values and set it to {DistribCore::Configuration.current} @@ -100,24 +137,24 @@ def initialize @first_test_picked_timeout = 10 * 60 # 10 minutes @tests_processing_stopped_timeout = 5 * 60 # 5 minutes @drb = { safe_level: 1 } - @drb_tcp_socket_connection_timeout = 5 # 5 seconds + @drb_tcp_socket_connection_timeout = 5 # in seconds @leader_connection_attempts = 200 self.timeout_strategy = :repush end - # Provider for tests to execute + # Provider for tests to execute. # # @return [Proc, Object#call] an object which responds to `#call` def tests_provider @tests_provider || raise(NotImplementedError) end - # Object to handle errors from workers + # Object to handle errors from workers. def error_handler @error_handler || raise(NotImplementedError) end - # Gives a timeout for a particular test based on `#test_timeout` + # Gives a timeout for a particular test based on `#test_timeout`. # # @see #test_timeout # diff --git a/distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb b/distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb index 32d2a9c..f0a55e4 100644 --- a/distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb +++ b/distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb @@ -1,10 +1,12 @@ -# A patch to reduce connection timeout on DRb Socket. -# @see https://rubydoc.info/stdlib/drb/DRb module DRb + # A monkey patch to reduce connection timeout on DRb Socket. # The following monkey-patch sets much lower value for connection timeout # By default it is over 2 minutes and it is causing a major worker shutdown # delay when the leader has finished already. + # + # @see https://rubydoc.info/stdlib/drb/DRb # @see https://rubydoc.info/stdlib/drb/DRb/DRbTCPSocket + # @see https://github.com/ruby/drb/blob/master/lib/drb/drb.rb class DRbTCPSocket # @param uri [String] # @param config [Hash] diff --git a/distrib-core/lib/distrib_core/distrib.rb b/distrib-core/lib/distrib_core/distrib.rb index 0bdf880..6be101a 100644 --- a/distrib-core/lib/distrib_core/distrib.rb +++ b/distrib-core/lib/distrib_core/distrib.rb @@ -8,7 +8,7 @@ def configure(...) configuration.instance_eval(...) end - # Set kind of the current instance + # Set kind of the current instance. # # @param kind [Symbol] `:leader` or `:worker` only def kind=(kind) diff --git a/distrib-core/lib/distrib_core/leader/error_handler.rb b/distrib-core/lib/distrib_core/leader/error_handler.rb index 6e03c05..f28d801 100644 --- a/distrib-core/lib/distrib_core/leader/error_handler.rb +++ b/distrib-core/lib/distrib_core/leader/error_handler.rb @@ -17,6 +17,9 @@ def initialize(exception_extractor) @exception_extractor = exception_extractor end + # Decides if the test should be retried. + # + # @return [TrueClass, FalseClass] def retry_test?(test, results, exception) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity return false if retries_per_test[test] >= retry_attempts @@ -43,6 +46,9 @@ def retry_test?(test, results, exception) # rubocop:disable Metrics/AbcSize, Met retried end + # Decides if the exception should be ignored. + # + # @return [TrueClass, FalseClass] def ignore_worker_failure?(exception) self.failed_workers_count += 1 diff --git a/distrib-core/lib/distrib_core/leader/queue_builder.rb b/distrib-core/lib/distrib_core/leader/queue_builder.rb index f6f82f6..3d11b3d 100644 --- a/distrib-core/lib/distrib_core/leader/queue_builder.rb +++ b/distrib-core/lib/distrib_core/leader/queue_builder.rb @@ -1,8 +1,7 @@ module DistribCore module Leader - # Helper that builds a list of the test files to execute sorted by average - # execution time descending. The order strategy is backed by - # https://en.wikipedia.org/wiki/Queueing_theory + # Helper that builds a list of the test files to execute. + # The order of files is controlled by used tests provider. module QueueBuilder # @return [Array] list of test files in the order they should be enqueued def self.tests diff --git a/distrib-core/lib/distrib_core/leader/queue_with_lease.rb b/distrib-core/lib/distrib_core/leader/queue_with_lease.rb index 6005d75..383cd72 100644 --- a/distrib-core/lib/distrib_core/leader/queue_with_lease.rb +++ b/distrib-core/lib/distrib_core/leader/queue_with_lease.rb @@ -3,7 +3,7 @@ module DistribCore module Leader - # Generic queue with lease. + # Generic queue-like, thread-safe container with lease. # # Additionally it keeps the time of the lease, allowing watchdog to return # (repush) timed out entries back to the queue. diff --git a/distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb b/distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb index 78d3487..07b377a 100644 --- a/distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb +++ b/distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb @@ -1,6 +1,7 @@ module DistribCore module Leader # Only retry if the error is different. + # This is a specialized strategy to manage retries of tests. class RetryOnDifferentErrorHandler < ErrorHandler def initialize(exception_extractor, retry_limit: 2, repeated_error_limit: 1) super(exception_extractor) diff --git a/distrib-core/lib/distrib_core/leader/watchdog.rb b/distrib-core/lib/distrib_core/leader/watchdog.rb index f354cc2..bfb4995 100644 --- a/distrib-core/lib/distrib_core/leader/watchdog.rb +++ b/distrib-core/lib/distrib_core/leader/watchdog.rb @@ -1,5 +1,3 @@ -require 'rainbow/refinement' - module DistribCore module Leader # A watchdog to observe the state of queue. @@ -8,8 +6,6 @@ module Leader # A thread watching over presence of the entries on the queue and lease # timeouts. Stops the {Leader} by stopping its DRb exposed service. class Watchdog # rubocop:disable Metrics/ClassLength - using Rainbow - def initialize(queue) @queue = queue @failed = false diff --git a/distrib-core/lib/distrib_core/metrics.rb b/distrib-core/lib/distrib_core/metrics.rb index ced5025..7fc0c78 100644 --- a/distrib-core/lib/distrib_core/metrics.rb +++ b/distrib-core/lib/distrib_core/metrics.rb @@ -2,7 +2,7 @@ module DistribCore # Collect metrics from Leader and Workers. module Metrics class << self - # Stores metrics + # Stores metrics. def report @report ||= { queue_exposed_at: nil, @@ -12,17 +12,17 @@ def report } end - # Records Leader is ready to serve tests + # Records when Leader was ready to serve tests. def queue_exposed report[:queue_exposed_at] = Time.now.to_i end - # Records first test was taken by a worker + # Records when first test was taken by a worker def test_taken report[:first_test_taken_at] ||= Time.now.to_i end - # Records when watchdog repushes files back to queue because of timeout + # Records when watchdog repushes files back to queue because of timeout. # # @param test [String] # @param timeout_in_seconds [Float] timeout which was exceeded diff --git a/distrib-core/lib/distrib_core/received_signals.rb b/distrib-core/lib/distrib_core/received_signals.rb index c8cff10..9e2d79e 100644 --- a/distrib-core/lib/distrib_core/received_signals.rb +++ b/distrib-core/lib/distrib_core/received_signals.rb @@ -9,7 +9,6 @@ class << self # @param sig [String] signal to trap # @example # ::DistribCore::ReceivedSignals.trap('INT') - # def trap(sig) Signal.trap(sig) do # Second SIGINT finishes the process immediately @@ -18,6 +17,7 @@ def trap(sig) puts 'Received second SIGINT. Exiting...' Kernel.exit(2) # 2 is exit code for SIGINT end + puts "Received #{sig}" signals.add(sig) end @@ -29,7 +29,7 @@ def any? end # @param sig [String] - # @return [TrueClass, FalseClass] `true` if signal `sig` was recieved + # @return [TrueClass, FalseClass] `true` if signal `sig` was received def received?(sig) signals.member?(sig) end diff --git a/distrib-core/lib/distrib_core/spec/configuration.rb b/distrib-core/lib/distrib_core/spec/configuration.rb index 448d9a5..8d6ff9a 100644 --- a/distrib-core/lib/distrib_core/spec/configuration.rb +++ b/distrib-core/lib/distrib_core/spec/configuration.rb @@ -4,6 +4,7 @@ config = DistribCore::Configuration.instance_variable_get(:@current) DistribCore::Configuration.instance_variable_set(:@current, nil) example.run + ensure DistribCore::Configuration.instance_variable_set(:@current, config) end diff --git a/distrib-core/lib/distrib_core/spec/distrib.rb b/distrib-core/lib/distrib_core/spec/distrib.rb index 38d33ce..bf9fb86 100644 --- a/distrib-core/lib/distrib_core/spec/distrib.rb +++ b/distrib-core/lib/distrib_core/spec/distrib.rb @@ -3,6 +3,7 @@ config = DistribCore::Configuration.instance_variable_get(:@current) DistribCore::Configuration.instance_variable_set(:@current, nil) example.run + ensure DistribCore::Configuration.instance_variable_set(:@current, config) end diff --git a/features-parser/.rubocop.yml b/features-parser/.rubocop.yml index 51510db..9ccccaa 100644 --- a/features-parser/.rubocop.yml +++ b/features-parser/.rubocop.yml @@ -1,6 +1,7 @@ AllCops: DisplayCopNames: true NewCops: enable + SuggestExtensions: false Style/StringLiterals: Enabled: true diff --git a/features-parser/README.md b/features-parser/README.md index 8fa4dcf..4e732fe 100644 --- a/features-parser/README.md +++ b/features-parser/README.md @@ -1,17 +1,45 @@ # FeaturesParser +This is an abstraction over [Gherkin](https://github.com/cucumber/gherkin/tree/main/ruby) gem. + ## Installation Add the gem to the application's Gemfile: ```ruby -gem 'features-parser', git: 'git@github.com:toptal/test-distrib.git', - glob: 'features-parser/*.gemspec' +git 'git@github.com:toptal/test-distrib.git', branch: 'main' do + gem 'features-parser' +end +``` + +## Usage + +```shell +bin/console +``` + +```ruby +feature_file = 'spec/support/some.feature' +catalog = FeaturesParser::Catalog.new +catalog.parse([feature_file]) + +catalog.names.take(3) +# => +# ["user-does-random-things/sending-as-a-guest-user", +# "user-does-random-things/staff-sends-feedback/user|john-doe-com", +# "user-does-random-things/staff-sends-feedback/moderator|agent-smith-com"] + +executable_paths = catalog.executable_paths_for(['user-does-random-things/sending-as-a-guest-user']) +# => ["spec/support/some.feature:10"] + +names = catalog.names_for(executable_paths) +# => ["user-does-random-things/sending-as-a-guest-user"] ``` ## Development After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. +Run `bundle exec rubocop` to check code style. ## Contributing diff --git a/rspec-distrib/.rubocop.yml b/rspec-distrib/.rubocop.yml index d5c877b..5fbf83a 100644 --- a/rspec-distrib/.rubocop.yml +++ b/rspec-distrib/.rubocop.yml @@ -6,6 +6,7 @@ require: AllCops: DisplayCopNames: true NewCops: enable + SuggestExtensions: false Exclude: - features/fixtures/**/* - coverage/**/* diff --git a/rspec-distrib/Gemfile.lock b/rspec-distrib/Gemfile.lock index da9f4d5..853c95d 100644 --- a/rspec-distrib/Gemfile.lock +++ b/rspec-distrib/Gemfile.lock @@ -7,6 +7,7 @@ PATH remote: . specs: rspec-distrib (0.0.1) + distrib-core (~> 0.0.1) rspec-core (~> 3.12) GEM @@ -20,8 +21,8 @@ GEM json (2.7.2) language_server-protocol (3.17.0.3) method_source (1.1.0) - parallel (1.24.0) - parser (3.3.1.0) + parallel (1.25.1) + parser (3.3.3.0) ast (~> 2.4.1) racc pry (0.14.2) @@ -34,8 +35,8 @@ GEM rainbow (3.1.1) rake (13.2.1) regexp_parser (2.9.2) - rexml (3.2.8) - strscan (>= 3.0.9) + rexml (3.3.0) + strscan rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -62,17 +63,8 @@ GEM unicode-display_width (>= 2.4.0, < 3.0) rubocop-ast (1.31.3) parser (>= 3.3.1.0) - rubocop-capybara (2.20.0) - rubocop (~> 1.41) - rubocop-factory_bot (2.25.1) - rubocop (~> 1.41) - rubocop-rspec (2.29.2) - rubocop (~> 1.40) - rubocop-capybara (~> 2.17) - rubocop-factory_bot (~> 2.22) - rubocop-rspec_rails (~> 2.28) - rubocop-rspec_rails (2.28.3) - rubocop (~> 1.40) + rubocop-rspec (3.0.1) + rubocop (~> 1.61) ruby-progressbar (1.13.0) simplecov (0.22.0) docile (~> 1.1) diff --git a/rspec-distrib/README.md b/rspec-distrib/README.md index c7942bf..b7b26d3 100644 --- a/rspec-distrib/README.md +++ b/rspec-distrib/README.md @@ -38,19 +38,15 @@ The queue is fault-tolerant, e.g. when a worker machine goes down or experiences a network partition, the spec file being executed is returned back to the queue on timeout, and later passed to another worker. -Spec files are served from the slowest (basing on previous builds results) to -fastest to reduce worker idle time, and reduce the risk of waiting for a long -spec file to execute in the end of the build. - ## Installation Add to the Gemfile: ```ruby -gem 'distrib-core', git: 'git@github.com:toptal/test-distrib.git', - glob: 'distrib-core/*.gemspec' -gem 'rspec-distrib', git: 'git@github.com:toptal/test-distrib.git', - glob: 'rspec-distrib/*.gemspec' +git 'git@github.com:toptal/test-distrib.git', branch: 'main' do + gem 'distrib-core', require: false, group: [:test] + gem 'rspec-distrib', require: false, group: [:test] +end ``` ## Running @@ -59,13 +55,11 @@ There is not much difference between running on local machine, or across the network. ```shell -$ rspec-distrib start | $ rspec-distrib join localhost +$ bundle exec rspec-distrib start | $ bundle exec rspec-distrib join 127.0.0.1 4386 files have been enqueued | .................*......F..._ Using seed 27792 | ``` -You may have to prefix the command with `bundle exec`. - ## Leader The Leader purpose is to serve spec file names one by one to workers, and @@ -73,7 +67,7 @@ aggregate the results of running those specs with RSpec. The following command: - builds a queue of spec files -- starts a watchdog thread (see more about watchdog in the 3nd stage) +- starts a watchdog thread (see more about watchdog in the 3rd stage) - exposes a Leader [DRb] server on all the network interfaces ```shell @@ -111,6 +105,55 @@ the execution results `report_file` back to the leader. ![Worker workflow](docs/worker.png) +## Development + +To start up development of the gem first make sure you could run the following commands without problems: + +```shell +bundle install +bundle exec rspec spec +bundle exec rspec features +bundle exec rubocop +bundle exec yardoc --fail-on-warning +``` + +Tests under `spec` are unit tests, while `features` are integration tests. + +Features could be used for manual tests while developing. +To proceed such manual test open following directory in **two separate console tabs**/windows: + +```shell +cd features/fixtures/specs/ +``` + +### Leader + +Pick features set (directory name) from [features/fixtures/specs](features/fixtures/specs). +Assuming we want to run `passing` features set: + +```shell +export RSPEC_DISTRIB_FOLDER=passing +``` + +Tune settings to have more time for manual tests: + +```shell +export RSPEC_DISTRIB_FEATURES_TEST_TIMEOUT=5 +export RSPEC_DISTRIB_FEATURES_FIRST_TEST_PICKED_TIMEOUT=30 +``` + +Start leader: + +```shell +bundle exec rspec-distrib start +``` + +### Worker + +```shell +bundle exec rspec-distrib join 127.0.0.1 +``` + ## Reports Workers are sending the example reports to the Leader immediately after running @@ -122,6 +165,8 @@ all the previous reports. `rspec-distrib` expects to find configuration in `.rspec-distrib` file which is loaded if it exists. Configuration is expected to be a Ruby file. +:information_source: See [rspec/distrib/configuration.rb](lib/rspec/distrib/configuration.rb) for the full list of options. + ### Spec files to execute Override default list of the spec files: @@ -186,7 +231,7 @@ class MyErrorHandlingStrategy end def ignore_worker_failure?(exception, context_description) - # return true to ingore the exception + # return true to ignore the exception end end ``` @@ -209,25 +254,19 @@ RSpec::Distrib.configure do |config| end ``` -Specify timeout per spec file. An object that responds to `call` and receives +To specify timeout per spec file use and object that responds to `call` and receives the spec file path as an argument. The proc returns the timeout in seconds. +This is also useful for cases where some specs have a timeout strategy and some +don't. ```ruby RSpec::Distrib.configure do |config| - config.timeout_proc = ->(spec_file) do + config.test_timeout = ->(spec_file) do 10 + 2 * average_execution_in_seconds(spec_file) end end ``` -If both `timeout_proc` and `test_timeout` are provided, `timeout_proc` -will take precedence, unless it returns a falsey value, in which case -it will fallback to `test_timeout`. -This is useful for cases where some specs have a timeout strategy and some -don't. - -### See lib/rspec/distrib/configuration.rb for the full list of options - ## RSpec configuration All RSpec configuration should be in `spec_helper.rb` or `rails_helper.rb`. @@ -287,9 +326,9 @@ It's port 8787, default for [DRb]. Yes, thread-safe data structures are used in the implementation, and access to non-thread-safe data structures is synchronized. -> Are the workers using the same seed the execute the specs? +> Are the workers using the same seed to execute the specs? -Yes. +Yes. Seed is set on the Leader side and passed to the workers. > Is it fault-tolerant? @@ -302,9 +341,13 @@ they crash. The Leader drops the connection when there's nothing left, and Workers shut down gracefully. -[DRb]: https://ruby-doc.org/stdlib-2.5.3/libdoc/drb/rdoc/DRb.html -[knapsack]: https://github.com/ArturT/knapsack -[parallel_tests]: https://github.com/grosser/parallel_tests +> What's the order of serving the spec files? + +Order is controlled by `tests_provider`. Default provided strategy is at [rspec/distrib/leader/tests_provider.rb](rspec-distrib/lib/rspec/distrib/leader/tests_provider.rb) - it's simplified one. + +In more advanced scenario spec files could be served from the slowest (basing on previous builds results) to +the fastest to reduce worker idle time, and reduce the risk of waiting for a long +spec file to execute in the end of the build. This could be achieved by implementing custom provider. ## Contributing @@ -313,3 +356,6 @@ Bug reports and pull requests are welcome [on GitHub](https://github.com/toptal/ ## License The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). + +[DRb]: https://ruby-doc.org/stdlib-2.5.3/libdoc/drb/rdoc/DRb.html +[parallel_tests]: https://github.com/grosser/parallel_tests diff --git a/rspec-distrib/features/feature_helper.rb b/rspec-distrib/features/feature_helper.rb index 1b89720..5a3dc9f 100644 --- a/rspec-distrib/features/feature_helper.rb +++ b/rspec-distrib/features/feature_helper.rb @@ -2,6 +2,6 @@ require_relative '../spec/spec_helper' -SimpleCov.add_filter 'features/support/' +SimpleCov.add_filter 'features/support/' if ENV['DISABLE_SIMPLECOV'] != '1' Dir[Pathname(__dir__).join('support', '**', '*.rb')].each { |f| require f } diff --git a/rspec-distrib/lib/rspec/distrib.rb b/rspec-distrib/lib/rspec/distrib.rb index 1b8ece4..b29accc 100644 --- a/rspec-distrib/lib/rspec/distrib.rb +++ b/rspec-distrib/lib/rspec/distrib.rb @@ -1,4 +1,4 @@ -require 'distrib_core' +require 'distrib-core' require 'rspec/distrib/configuration' # @see https://github.com/rspec/rspec diff --git a/rspec-distrib/lib/rspec/distrib/configuration.rb b/rspec-distrib/lib/rspec/distrib/configuration.rb index b8e0606..7969fce 100644 --- a/rspec-distrib/lib/rspec/distrib/configuration.rb +++ b/rspec-distrib/lib/rspec/distrib/configuration.rb @@ -24,7 +24,7 @@ module Distrib # end # # Or set your own logic for retries. - # It should respond to `#retry_test?`, `#ignore_worker_failure?` methods + # It should respond to `#retry_test?`, `#ignore_worker_failure?` methods. # # RSpec::Distrib.configure do |config| # config.error_handler = MyErrorHandler @@ -46,29 +46,29 @@ module Distrib # end # # Set how long leader will wait before first spec processed by workers. Leader will exit if - # no specs picked in this period + # no specs picked in this period. # # RSpec::Distrib.configure do |config| # config.first_test_picked_timeout = 10*60 # 10 minutes # end # # Set how long leader will wait if workers stopped processing the queue. Leader will exit if - # no specs picked in this period + # no specs picked in this period. # # RSpec::Distrib.configure do |config| # config.tests_processing_stopped_timeout = 5*60 # 5 minutes # end # # Specify which formatters you want to use using `add_leader_formatter` or `add_worker_formatter` methods. - # See `RSpec.configuration.add_formatter` for more info + # See `RSpec.configuration.add_formatter` for more info. # # RSpec::Distrib.configure do |config| # config.add_leader_formatter('html', 'summary.html') # add HTML formatter which writes to 'summary.html' file # config.add_worker_formatter('progress') # add progress formatter (prints dots to the console) # end # - # Specify custom options for DRb service. Defaults are `{ safe_level: 1 }` - # See `DRb::DRbServer.new` for complete list + # Specify custom options for DRb service. Defaults are `{ safe_level: 1 }`. + # See `DRb::DRbServer.new` for complete list. # # RSpec::Distrib.configure do |config| # config.drb = {safe_level: 1} @@ -85,7 +85,7 @@ module Distrib # class Configuration include ::DistribCore::Configuration - # Sets RSpec's `--color` option for workers with "force" mode, rewriting existing one + # Sets RSpec's `--color` option for workers with "force" mode, rewriting existing one. # Possible values: :on, :off; by default it's :automatic # See https://github.com/rspec/rspec-core/blob/7510b747cdb028dea4feb56cef8062cea14640a5/lib/rspec/core/configuration.rb#L937 attr_accessor :worker_color_mode diff --git a/rspec-distrib/lib/rspec/distrib/example_group.rb b/rspec-distrib/lib/rspec/distrib/example_group.rb index 797ce97..fbc6e5f 100644 --- a/rspec-distrib/lib/rspec/distrib/example_group.rb +++ b/rspec-distrib/lib/rspec/distrib/example_group.rb @@ -4,7 +4,8 @@ module RSpec module Distrib # Helper to proxy getter methods to metadata attributes. module DelegateToMetadata - # Defines methods that fetch attributes from metadata hash + # Defines methods that fetch attributes from metadata hash. + # # @param keys [Array] def delegate_to_metadata(*keys) keys.each { |key| define_method(key) { @metadata[key] } } diff --git a/rspec-distrib/lib/rspec/distrib/leader.rb b/rspec-distrib/lib/rspec/distrib/leader.rb index 5ae856c..4724425 100644 --- a/rspec-distrib/lib/rspec/distrib/leader.rb +++ b/rspec-distrib/lib/rspec/distrib/leader.rb @@ -12,13 +12,13 @@ module Distrib # Transport used is [DRb](https://rubydoc.info/stdlib/drb/DRb) class Leader # rubocop:disable Metrics/ClassLength include ::DistribCore::Leader - # Used to interpolate with leader ip in order to generate the actual DRb server URL + # Used to interpolate with leader ip in order to generate the actual DRb server URL. DRB_SERVER_URL = 'druby://%s:8787'.freeze - # We can't calculate total amount of examples. But we need to provide a big number to prevent warnings + # We can't calculate total amount of examples. But we need to provide a big number to prevent warnings. FAKE_TOTAL_EXAMPLES_COUNT = 1_000_000_000 class << self - # Starts the DRb server and Watchdog thread + # Starts the DRb server and Watchdog thread. # # @param seed [Integer] a seed for workers to randomize order of examples # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity @@ -74,8 +74,8 @@ def print_failure_status(reporter, watchdog, leader, queue, count_mismatch) # rubocop:enable Metrics/AbcSize end - # Object shared through DRb is open for any calls. Including eval calls - # A simple way to prevent it - undef + # Object shared through DRb is open for any calls. Including eval calls. + # A simple way to prevent it - undef. undef :instance_eval undef :instance_exec @@ -88,7 +88,8 @@ def initialize(queue, reporter, seed) logger.info "Using seed #{@seed}" end - # Get the next spec from the queue + # Get the next spec from the queue. + # # @return [String] spec file name # @example # leader.next_file_to_run # => 'spec/services/user_service_spec.rb' @@ -100,7 +101,7 @@ def initialize(queue, reporter, seed) end end - # Report example group results for a spec file + # Report example group results for a spec file. # # @param file_path [String] ex: './spec/services/user_service_spec.rb' # @param example_groups [Array] @@ -184,7 +185,6 @@ def handle_non_example_exception def log_completed_percent # rubocop:disable Metrics/AbcSize @logged_percents ||= [] log_every = 10 - completed_percent = (queue.completed_size.to_f / (queue.size + queue.completed_size) * 100).to_i bucket = completed_percent / log_every * log_every # convert 35 to 30 diff --git a/rspec-distrib/lib/rspec/distrib/leader/reporter.rb b/rspec-distrib/lib/rspec/distrib/leader/reporter.rb index b03d8ba..ac02e60 100644 --- a/rspec-distrib/lib/rspec/distrib/leader/reporter.rb +++ b/rspec-distrib/lib/rspec/distrib/leader/reporter.rb @@ -6,7 +6,7 @@ module RSpec module Distrib class Leader - # RSpec reporter local to the leader, but reachable by Workers. + # RSpec reporter local to the Leader, but reachable by Workers. # Used to accumulate the example execution results from worker machines. class Reporter # Failed statuses for an example @@ -39,12 +39,12 @@ def report(example_group, will_be_retried: false) reporter.example_group_finished(example_group) end - # Print the final results of the test suite + # Print the final results of the test suite. def finish reporter.finish end - # Notifies RSpec about exceptions unrelated to an example in order to halt execution + # Notifies RSpec about exceptions unrelated to an example in order to halt execution. # # @param exception [Exception] def notify_non_example_exception(exception, context_description) @@ -67,7 +67,7 @@ def init_reporter RSpec.configuration.reporter end - # Adds example to report. Notifies formatters + # Adds example to report. Notifies formatters. def report_example(example_result, will_be_retried:) status = example_result.execution_result.status.to_s @@ -75,7 +75,7 @@ def report_example(example_result, will_be_retried:) if will_be_retried # We retry the whole file, but we only want to - # report the specs that actually failed and cause the retry + # report the specs that actually failed and cause the retry. return unless FAILED_STATUSES.include?(status) reporter.publish(:example_will_be_retried, example: example_result) diff --git a/rspec-distrib/lib/rspec/distrib/worker.rb b/rspec-distrib/lib/rspec/distrib/worker.rb index f7c0c86..e836fe2 100644 --- a/rspec-distrib/lib/rspec/distrib/worker.rb +++ b/rspec-distrib/lib/rspec/distrib/worker.rb @@ -3,7 +3,7 @@ module RSpec module Distrib - # Wrapper around {RSpec::Distrib::RSpecRunner} + # Wrapper around {RSpec::Distrib::RSpecRunner}. module Worker # Start a worker instance with a given leader ip. # diff --git a/rspec-distrib/lib/rspec/distrib/worker/configuration.rb b/rspec-distrib/lib/rspec/distrib/worker/configuration.rb index 07c7918..b689bbb 100644 --- a/rspec-distrib/lib/rspec/distrib/worker/configuration.rb +++ b/rspec-distrib/lib/rspec/distrib/worker/configuration.rb @@ -8,7 +8,7 @@ class Configuration < RSpec::Core::Configuration # @return [DRbObject(RSpec::Distrib::Leader)] attr_accessor :leader - # Overridden method which wraps original reporter with {LeaderReporter} + # Overridden method which wraps original reporter with {LeaderReporter}. # @return RSpec::Core::Formatters::Loader def formatter_loader @formatter_loader ||= begin @@ -18,7 +18,7 @@ def formatter_loader end end - # Always true because seed comes from leader + # Always true because seed comes from leader. def seed_used? true end diff --git a/rspec-distrib/lib/rspec/distrib/worker/leader_reporter.rb b/rspec-distrib/lib/rspec/distrib/worker/leader_reporter.rb index 3f35717..a08ac04 100644 --- a/rspec-distrib/lib/rspec/distrib/worker/leader_reporter.rb +++ b/rspec-distrib/lib/rspec/distrib/worker/leader_reporter.rb @@ -4,7 +4,7 @@ module RSpec module Distrib module Worker # @api private - # Custom reporter to notify leader about non_example_exception + # Custom reporter to notify leader about non_example_exception. class LeaderReporter < SimpleDelegator # @param leader [DRbObject(RSpec::Distrib::Leader)] def initialize(leader, *) @@ -12,7 +12,7 @@ def initialize(leader, *) @leader = leader end - # Calls original behaviour and notifies leader + # Calls original behaviour and notifies leader. # @param exception [Exception] def notify_non_example_exception(exception, context_description) super diff --git a/rspec-distrib/lib/rspec/distrib/worker/rspec_runner.rb b/rspec-distrib/lib/rspec/distrib/worker/rspec_runner.rb index 1a188af..c287163 100644 --- a/rspec-distrib/lib/rspec/distrib/worker/rspec_runner.rb +++ b/rspec-distrib/lib/rspec/distrib/worker/rspec_runner.rb @@ -14,7 +14,7 @@ module RSpec module Distrib module Worker - # Modified RSpec runner to consume files from the leader + # Modified RSpec runner to consume files from the leader. # # @see https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/runner.rb class RSpecRunner < RSpec::Core::Runner # rubocop:disable Metrics/ClassLength @@ -37,8 +37,9 @@ def initialize(leader) # rubocop:disable Metrics/MethodLength handle_configuration_failure do @options = RSpec::Core::ConfigurationOptions.new(["--seed=#{@seed}"]) - # as long as there is this assignment to global variable + # As long as there is this assignment to global variable # the test have to restore RSpec.configuration after the example + # # see `around` block in spec/rspec/distrib/worker/rspec_runner_spec.rb @configuration = RSpec.configuration = RSpec::Distrib::Worker::Configuration.new @configuration.leader = leader @@ -59,14 +60,14 @@ def run(*) # rubocop:disable Metrics/MethodLength handle_configuration_failure do @configuration.reporter.report(Leader::FAKE_TOTAL_EXAMPLES_COUNT) do |reporter| @configuration.with_suite_hooks do - # Disable handler since consume_queue has it's own handler + # Disable handler since consume_queue has it's own handler. @handle_configuration_failure = false consume_queue(reporter) end end end - persist_example_statuses + persist_example_statuses return ::DistribCore::ReceivedSignals.exit_code if received_any_signal? success && !world.non_example_failure ? 0 : @configuration.failure_exit_code @@ -99,8 +100,8 @@ def consume_queue(reporter) @success &= world.ordered_example_groups.map { |example_group| example_group.run(reporter) }.all? - # Should not send a possible broken report for the leader - # A report can be broken because other services (e.g. Redis, Elasticsearch) could have already terminated + # Should not send a possible broken report for the leader. + # A report can be broken because other services (e.g. Redis, Elasticsearch) could have already terminated. break if received_term? || received_force_int? report_file_to_leader(file_path, world.ordered_example_groups) @@ -108,7 +109,7 @@ def consume_queue(reporter) break if received_int? || world.non_example_failure end rescue DRb::DRbConnError - # It raises when Leader is disconnected = a.k.a. queue is empty + # It raises when Leader is disconnected = a.k.a. queue is empty. logger.info 'Disconnected from leader, finishing' rescue Exception => e # rubocop:disable Lint/RescueException # TODO: I'm unsure about this rescue, but we need to report all cases to leader @@ -185,9 +186,11 @@ def load_spec_file(path) world.example_groups.clear @options = RSpec::Core::ConfigurationOptions.new(["--seed=#{@seed}", path]) @options.configure(@configuration) + if RSpec::Distrib.configuration.worker_color_mode @configuration.force(color_mode: RSpec::Distrib.configuration.worker_color_mode) end + @configuration.load_spec_files end diff --git a/rspec-distrib/rspec-distrib.gemspec b/rspec-distrib/rspec-distrib.gemspec index 9b737de..5c38480 100644 --- a/rspec-distrib/rspec-distrib.gemspec +++ b/rspec-distrib/rspec-distrib.gemspec @@ -16,6 +16,7 @@ Gem::Specification.new do |s| s.bindir = 'exe' s.executables = ['rspec-distrib'] + s.add_dependency 'distrib-core', '~> 0.0.1' s.add_dependency 'rspec-core', '~> 3.12' s.metadata['rubygems_mfa_required'] = 'true' diff --git a/rspec-distrib/spec/rspec/distrib/leader/rspec_helper_spec.rb b/rspec-distrib/spec/rspec/distrib/leader/rspec_helper_spec.rb index 7e64674..667afbe 100644 --- a/rspec-distrib/spec/rspec/distrib/leader/rspec_helper_spec.rb +++ b/rspec-distrib/spec/rspec/distrib/leader/rspec_helper_spec.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -RSpec.describe RSpec::Distrib::Leader::RSpecHelper do # rubocop:disable RSpec/FilePath, RSpec/SpecFilePathFormat +RSpec.describe RSpec::Distrib::Leader::RSpecHelper do # rubocop:disable RSpec/SpecFilePathFormat describe '.failures_of' do it do errors = [ diff --git a/rspec-distrib/spec/rspec/distrib/worker/rspec_runner_spec.rb b/rspec-distrib/spec/rspec/distrib/worker/rspec_runner_spec.rb index 83a5084..820d6ed 100644 --- a/rspec-distrib/spec/rspec/distrib/worker/rspec_runner_spec.rb +++ b/rspec-distrib/spec/rspec/distrib/worker/rspec_runner_spec.rb @@ -2,7 +2,7 @@ require 'tempfile' -RSpec.describe RSpec::Distrib::Worker::RSpecRunner do # rubocop:disable RSpec/FilePath, RSpec/SpecFilePathFormat +RSpec.describe RSpec::Distrib::Worker::RSpecRunner do # rubocop:disable RSpec/SpecFilePathFormat include RSpec::Support::InSubProcess # restore the original value to prevent leaking leader double to other examples