From 89d46ddfa72e9beb75e2363861cdb6d3195e3f0f Mon Sep 17 00:00:00 2001 From: Yuri Smirnov Date: Fri, 6 Sep 2024 18:14:32 +0600 Subject: [PATCH] Add jobs helpers (#33) --- .github/workflows/test.yml | 2 +- .rubocop.yml | 2 +- Gemfile | 2 +- Gemfile.lock | 209 +++++++++--------- lib/umbrellio_utils.rb | 5 +- lib/umbrellio_utils/control.rb | 2 +- lib/umbrellio_utils/database.rb | 12 +- lib/umbrellio_utils/formatting.rb | 2 +- lib/umbrellio_utils/jobs.rb | 86 +++++++ lib/umbrellio_utils/misc.rb | 2 +- lib/umbrellio_utils/parsing.rb | 16 +- lib/umbrellio_utils/vault.rb | 2 +- lib/umbrellio_utils/version.rb | 2 +- spec/support/sequel_patches.rb | 2 +- spec/umbrellio_utils/jobs_spec.rb | 123 +++++++++++ spec/umbrellio_utils/request_wrapper_spec.rb | 2 +- .../tiny_json_formatter_spec.rb | 2 +- umbrellio_utils.gemspec | 2 +- 18 files changed, 344 insertions(+), 131 deletions(-) create mode 100644 lib/umbrellio_utils/jobs.rb create mode 100644 spec/umbrellio_utils/jobs_spec.rb diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d50b9c..3f0ab81 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: ["3.0", "3.1", "3.2", "3.3"] + ruby: ["3.1", "3.2", "3.3"] services: postgres: diff --git a/.rubocop.yml b/.rubocop.yml index 8332173..9a3e166 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -3,7 +3,7 @@ inherit_gem: AllCops: DisplayCopNames: true - TargetRubyVersion: 3.0 + TargetRubyVersion: 3.1 SuggestExtensions: false Naming/MethodParameterName: diff --git a/Gemfile b/Gemfile index 5c90f84..797fd4c 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ gem "activesupport" gem "bundler" gem "ci-helper" gem "http" -gem "net-pop", github: "ruby/net-pop" # See https://stackoverflow.com/questions/78617432/strange-bundle-update-issue-disappearing-net-pop-0-1-2-dependency +gem "net-pop" gem "nokogiri" gem "nori" gem "pg" diff --git a/Gemfile.lock b/Gemfile.lock index 03a5bb9..d55b428 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,93 +1,82 @@ -GIT - remote: https://github.com/ruby/net-pop.git - revision: e8d0afe2773b9eb6a23c39e9e437f6fc0fc7c733 - specs: - net-pop (0.1.2) - net-protocol - PATH remote: . specs: - umbrellio-utils (1.4.0) + umbrellio-utils (1.5.0) memery (~> 1) GEM remote: https://rubygems.org/ specs: - actioncable (7.1.3.4) - actionpack (= 7.1.3.4) - activesupport (= 7.1.3.4) + actioncable (7.2.1) + actionpack (= 7.2.1) + activesupport (= 7.2.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.3.4) - actionpack (= 7.1.3.4) - activejob (= 7.1.3.4) - activerecord (= 7.1.3.4) - activestorage (= 7.1.3.4) - activesupport (= 7.1.3.4) - mail (>= 2.7.1) - net-imap - net-pop - net-smtp - actionmailer (7.1.3.4) - actionpack (= 7.1.3.4) - actionview (= 7.1.3.4) - activejob (= 7.1.3.4) - activesupport (= 7.1.3.4) - mail (~> 2.5, >= 2.5.4) - net-imap - net-pop - net-smtp + actionmailbox (7.2.1) + actionpack (= 7.2.1) + activejob (= 7.2.1) + activerecord (= 7.2.1) + activestorage (= 7.2.1) + activesupport (= 7.2.1) + mail (>= 2.8.0) + actionmailer (7.2.1) + actionpack (= 7.2.1) + actionview (= 7.2.1) + activejob (= 7.2.1) + activesupport (= 7.2.1) + mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.1.3.4) - actionview (= 7.1.3.4) - activesupport (= 7.1.3.4) + actionpack (7.2.1) + actionview (= 7.2.1) + activesupport (= 7.2.1) nokogiri (>= 1.8.5) racc - rack (>= 2.2.4) + rack (>= 2.2.4, < 3.2) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.3.4) - actionpack (= 7.1.3.4) - activerecord (= 7.1.3.4) - activestorage (= 7.1.3.4) - activesupport (= 7.1.3.4) + useragent (~> 0.16) + actiontext (7.2.1) + actionpack (= 7.2.1) + activerecord (= 7.2.1) + activestorage (= 7.2.1) + activesupport (= 7.2.1) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.3.4) - activesupport (= 7.1.3.4) + actionview (7.2.1) + activesupport (= 7.2.1) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.3.4) - activesupport (= 7.1.3.4) + activejob (7.2.1) + activesupport (= 7.2.1) globalid (>= 0.3.6) - activemodel (7.1.3.4) - activesupport (= 7.1.3.4) - activerecord (7.1.3.4) - activemodel (= 7.1.3.4) - activesupport (= 7.1.3.4) + activemodel (7.2.1) + activesupport (= 7.2.1) + activerecord (7.2.1) + activemodel (= 7.2.1) + activesupport (= 7.2.1) timeout (>= 0.4.0) - activestorage (7.1.3.4) - actionpack (= 7.1.3.4) - activejob (= 7.1.3.4) - activerecord (= 7.1.3.4) - activesupport (= 7.1.3.4) + activestorage (7.2.1) + actionpack (= 7.2.1) + activejob (= 7.2.1) + activerecord (= 7.2.1) + activesupport (= 7.2.1) marcel (~> 1.0) - activesupport (7.1.3.4) + activesupport (7.2.1) base64 bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) amazing_print (1.6.0) @@ -105,12 +94,12 @@ GEM umbrellio-sequel-plugins (~> 0.14) coderay (1.1.3) colorize (1.1.0) - concurrent-ruby (1.3.3) + concurrent-ruby (1.3.4) connection_pool (2.4.1) crass (1.0.6) date (3.3.4) diff-lcs (1.5.1) - docile (1.4.0) + docile (1.4.1) domain_name (0.6.20240107) drb (2.2.1) dry-inflector (1.1.0) @@ -130,22 +119,23 @@ GEM http-cookie (~> 1.0) http-form_data (~> 2.2) llhttp-ffi (~> 0.5.0) - http-cookie (1.0.6) + http-cookie (1.0.7) domain_name (~> 0.5) http-form_data (2.3.0) i18n (1.14.5) concurrent-ruby (~> 1.0) io-console (0.7.2) - irb (1.13.2) + irb (1.14.0) rdoc (>= 4.0.0) reline (>= 0.4.2) json (2.7.2) - lamian (1.9.0) + lamian (1.10.0) rails (>= 4.2) language_server-protocol (3.17.0.3) llhttp-ffi (0.5.0) ffi-compiler (~> 1.0) rake (~> 13.0) + logger (1.6.1) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -160,46 +150,47 @@ GEM method_source (1.1.0) mini_mime (1.1.5) mini_portile2 (2.8.7) - minitest (5.24.1) - mutex_m (0.2.0) - net-imap (0.4.14) + minitest (5.25.1) + net-imap (0.4.16) date net-protocol + net-pop (0.1.2) + net-protocol net-protocol (0.2.2) timeout net-smtp (0.5.0) net-protocol nio4r (2.7.3) - nokogiri (1.16.6) + nokogiri (1.16.7) mini_portile2 (~> 2.8.2) racc (~> 1.4) - nokogiri (1.16.6-arm64-darwin) + nokogiri (1.16.7-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.6-x86_64-darwin) + nokogiri (1.16.7-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.6-x86_64-linux) + nokogiri (1.16.7-x86_64-linux) racc (~> 1.4) - nori (2.7.0) + nori (2.7.1) bigdecimal - parallel (1.25.1) - parser (3.3.3.0) + parallel (1.26.3) + parser (3.3.5.0) ast (~> 2.4.1) racc - pg (1.5.6) + pg (1.5.7) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) psych (5.1.2) stringio - public_suffix (6.0.0) + public_suffix (6.0.1) rabbit_messaging (0.15.0) bunny (~> 2.0) lamian rails (>= 5.2) sneakers (~> 2.0) tainbox - racc (1.8.0) - rack (3.1.6) + racc (1.8.1) + rack (3.1.7) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) @@ -207,20 +198,20 @@ GEM rackup (2.1.0) rack (>= 3) webrick (~> 1.8) - rails (7.1.3.4) - actioncable (= 7.1.3.4) - actionmailbox (= 7.1.3.4) - actionmailer (= 7.1.3.4) - actionpack (= 7.1.3.4) - actiontext (= 7.1.3.4) - actionview (= 7.1.3.4) - activejob (= 7.1.3.4) - activemodel (= 7.1.3.4) - activerecord (= 7.1.3.4) - activestorage (= 7.1.3.4) - activesupport (= 7.1.3.4) + rails (7.2.1) + actioncable (= 7.2.1) + actionmailbox (= 7.2.1) + actionmailer (= 7.2.1) + actionpack (= 7.2.1) + actiontext (= 7.2.1) + actionview (= 7.2.1) + activejob (= 7.2.1) + activemodel (= 7.2.1) + activerecord (= 7.2.1) + activestorage (= 7.2.1) + activesupport (= 7.2.1) bundler (>= 1.15.0) - railties (= 7.1.3.4) + railties (= 7.2.1) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest @@ -228,10 +219,10 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.1.3.4) - actionpack (= 7.1.3.4) - activesupport (= 7.1.3.4) - irb + railties (7.2.1) + actionpack (= 7.2.1) + activesupport (= 7.2.1) + irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) @@ -242,17 +233,16 @@ GEM rdoc (6.7.0) psych (>= 4.0.0) regexp_parser (2.9.2) - reline (0.5.9) + reline (0.5.10) io-console (~> 0.5) - rexml (3.3.1) - strscan + rexml (3.3.7) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) + rspec-core (3.13.1) rspec-support (~> 3.13.0) - rspec-expectations (3.13.1) + rspec-expectations (3.13.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-json_matcher (0.2.0) @@ -273,7 +263,7 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.3) + rubocop-ast (1.32.3) parser (>= 3.3.1.0) rubocop-capybara (2.21.0) rubocop (~> 1.41) @@ -307,10 +297,11 @@ GEM rubocop (~> 1.0) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) + securerandom (0.3.1) self_data (1.3.0) - semantic_logger (4.15.0) + semantic_logger (4.16.0) concurrent-ruby (~> 1.0) - sequel (5.82.0) + sequel (5.84.0) bigdecimal sequel-batches (2.0.2) sequel @@ -335,8 +326,6 @@ GEM rbtree set (~> 1.0) stringio (3.1.1) - strscan (3.1.0) - symbiont-ruby (0.7.0) table_sync (6.5.0) memery rabbit_messaging (~> 0.13) @@ -344,21 +333,21 @@ GEM self_data tainbox (2.1.2) activesupport - thor (1.3.1) + thor (1.3.2) timecop (0.9.10) timeout (0.4.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - umbrellio-sequel-plugins (0.15.0.198) + umbrellio-sequel-plugins (0.16.0.211) sequel - symbiont-ruby unicode-display_width (2.5.0) + useragent (0.16.10) webrick (1.8.1) websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - yard (0.9.36) - zeitwerk (2.6.16) + yard (0.9.37) + zeitwerk (2.6.18) PLATFORMS arm64-darwin-20 @@ -373,7 +362,7 @@ DEPENDENCIES bundler ci-helper http - net-pop! + net-pop nokogiri nori pg @@ -393,4 +382,4 @@ DEPENDENCIES yard BUNDLED WITH - 2.5.14 + 2.5.18 diff --git a/lib/umbrellio_utils.rb b/lib/umbrellio_utils.rb index 2eb0972..d7da4ce 100644 --- a/lib/umbrellio_utils.rb +++ b/lib/umbrellio_utils.rb @@ -41,8 +41,8 @@ def default_settings } end - def synchronize(&block) - GLOBAL_MUTEX.owned? ? yield : GLOBAL_MUTEX.synchronize(&block) + def synchronize(&) + GLOBAL_MUTEX.owned? ? yield : GLOBAL_MUTEX.synchronize(&) end end @@ -53,6 +53,7 @@ def synchronize(&block) require_relative "umbrellio_utils/database" require_relative "umbrellio_utils/formatting" require_relative "umbrellio_utils/http_client" +require_relative "umbrellio_utils/jobs" require_relative "umbrellio_utils/misc" require_relative "umbrellio_utils/parsing" require_relative "umbrellio_utils/passwords" diff --git a/lib/umbrellio_utils/control.rb b/lib/umbrellio_utils/control.rb index b623a52..e0d833a 100644 --- a/lib/umbrellio_utils/control.rb +++ b/lib/umbrellio_utils/control.rb @@ -21,7 +21,7 @@ def run_in_interval(interval, key:) def retry_on_unique_violation( times: Float::INFINITY, retry_on_all_constraints: false, checked_constraints: [], &block ) - retry_on(Sequel::UniqueConstraintViolation, times: times) do + retry_on(Sequel::UniqueConstraintViolation, times:) do DB.transaction(savepoint: true, &block) rescue Sequel::UniqueConstraintViolation => e constraint_name = Database.get_violated_constraint_name(e) diff --git a/lib/umbrellio_utils/database.rb b/lib/umbrellio_utils/database.rb index fe30924..f45cc20 100644 --- a/lib/umbrellio_utils/database.rb +++ b/lib/umbrellio_utils/database.rb @@ -7,8 +7,8 @@ module Database HandledConstaintError = Class.new(StandardError) InvalidPkError = Class.new(StandardError) - def handle_constraint_error(constraint_name, &block) - DB.transaction(savepoint: true, &block) + def handle_constraint_error(constraint_name, &) + DB.transaction(savepoint: true, &) rescue Sequel::UniqueConstraintViolation => e if constraint_name.to_s == get_violated_constraint_name(e) raise HandledConstaintError @@ -23,7 +23,7 @@ def get_violated_constraint_name(exception) end def each_record(dataset, primary_key: nil, **options, &block) - primary_key = primary_key_from(dataset, primary_key: primary_key) + primary_key = primary_key_from(dataset, primary_key:) with_temp_table(dataset, **options) do |ids| rows = ids.map { |id| row(id.is_a?(Hash) ? id.values : [id]) } @@ -47,11 +47,11 @@ def with_temp_table( primary_key: nil, temp_table_name: nil ) - primary_key = primary_key_from(dataset, primary_key: primary_key) + primary_key = primary_key_from(dataset, primary_key:) sleep_interval = sleep_interval_from(sleep) temp_table_name = create_temp_table( - dataset, primary_key: primary_key, temp_table_name: temp_table_name&.to_sym + dataset, primary_key:, temp_table_name: temp_table_name&.to_sym ) pk_set = [] @@ -77,7 +77,7 @@ def create_temp_table(dataset, primary_key: nil, temp_table_name: nil) temp_table_name ||= :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}" return temp_table_name if DB.table_exists?(temp_table_name) - primary_key = primary_key_from(dataset, primary_key: primary_key) + primary_key = primary_key_from(dataset, primary_key:) DB.create_table(temp_table_name, unlogged: true) do primary_key.each do |field| diff --git a/lib/umbrellio_utils/formatting.rb b/lib/umbrellio_utils/formatting.rb index 8e00005..9f4da81 100644 --- a/lib/umbrellio_utils/formatting.rb +++ b/lib/umbrellio_utils/formatting.rb @@ -58,7 +58,7 @@ def encode_key(key) end def to_date_part_string(part) - format("%02d", part: part) + format("%02d", part:) end # diff --git a/lib/umbrellio_utils/jobs.rb b/lib/umbrellio_utils/jobs.rb new file mode 100644 index 0000000..86f8c2c --- /dev/null +++ b/lib/umbrellio_utils/jobs.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module UmbrellioUtils::Jobs + extend self + + Worker = Struct.new(:name) + Capsule = Struct.new(:name, :worker, :weight) + Queue = Struct.new(:name, :capsule, :weight) + Entry = Struct.new(:capsule, :queues, :concurrency) + + def workers + @workers ||= [] + end + + def capsules + @capsules ||= [] + end + + def queues + @queues ||= [] + end + + def register_worker(name) + workers << Worker.new(name) + end + + def register_capsule(name, worker: :default, weight: 1) + workers.find { |x| x.name == worker } or raise "Worker not found: #{worker.inspect}" + capsules << Capsule.new(name, worker, weight) + end + + def register_queue(name, capsule: :default, weight: 1) + capsules.find { |x| x.name == capsule } or raise "Capsule not found: #{capsule.inspect}" + queues << Queue.new(name, capsule, weight) + end + + def retry_interval(error_count, min_interval:, max_interval:) + interval = min_interval * (1.3**(error_count - 3)) + interval.clamp(min_interval, max_interval).round + end + + def configure_capsules!(config, priority_level:, max_concurrency:) + entries = capsules_for(priority_level, max_concurrency) + + unless entries.find { |x| x.capsule == :default } + entries.last.capsule = :default # Default capsule should always be present in sidekiq + end + + entries.each do |entry| + config.capsule(entry.capsule) do |capsule| + capsule.queues = entry.queues + capsule.concurrency = entry.concurrency + end + end + end + + def capsules_for(worker, max_concurrency) + capsules = self.capsules.select do |capsule| + next unless capsule.worker.to_s == worker.underscore.to_s + next unless queues.any? { |queue| queue.capsule == capsule.name } + true + end + + total_weight = capsules.sum(&:weight) + + result = capsules.filter_map do |capsule| + weight_coef = capsule.weight / total_weight.to_f + concurrency = (max_concurrency * weight_coef).to_i + concurrency = 1 unless concurrency > 1 + queues = self.queues.select { |x| x.capsule == capsule.name }.map { |x| [x.name, x.weight] } + Entry.new(capsule.name, queues, concurrency) + end + + raise "No queues found for worker #{worker.inspect}" if result.empty? + + result + end + + def validate_queue_name!(queue_name) + found = queues.any? do |queue| + queue.name.to_s == queue_name.to_s + end + + raise "Unknown queue: #{queue_name.inspect}" unless found + end +end diff --git a/lib/umbrellio_utils/misc.rb b/lib/umbrellio_utils/misc.rb index e9f3020..98f3986 100644 --- a/lib/umbrellio_utils/misc.rb +++ b/lib/umbrellio_utils/misc.rb @@ -13,7 +13,7 @@ def table_sync(scope, delay: 1, routing_key: nil) TableSync::Publishing::Batch.new( object_class: model_class, original_attributes: batch_for_sync.map(&:values), - routing_key: routing_key, + routing_key:, ).publish_now sleep delay diff --git a/lib/umbrellio_utils/parsing.rb b/lib/umbrellio_utils/parsing.rb index 4f0f90b..17c1b6c 100644 --- a/lib/umbrellio_utils/parsing.rb +++ b/lib/umbrellio_utils/parsing.rb @@ -20,7 +20,7 @@ def parse_xml(xml, remove_attributes: true, snakecase: true) xml.remove_namespaces! xml.xpath("//@*").remove if remove_attributes - tags_converter = snakecase ? -> (tag) { tag.snakecase.to_sym } : -> (tag) { tag.to_sym } + tags_converter = snakecase ? -> (tag) { snakecase(tag).to_sym } : -> (tag) { tag.to_sym } nori = Nori.new(convert_tags_to: tags_converter, convert_dashes_to_underscores: false) nori.parse(xml.to_xml(save_with: Nokogiri::XML::Node::SaveOptions::NO_DECLARATION)) end @@ -72,5 +72,19 @@ def sanitize_phone(string, e164_format: false) phone.sanitized end + + private + + # See https://github.com/savonrb/nori/blob/main/lib/nori/string_utils.rb + def snakecase(string) + str = string.dup + str.gsub!("::", "/") + str.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2') + str.gsub!(/([a-z\d])([A-Z])/, '\1_\2') + str.tr!(".", "_") + str.tr!("-", "_") + str.downcase! + str + end end end diff --git a/lib/umbrellio_utils/vault.rb b/lib/umbrellio_utils/vault.rb index bba247e..1fc9761 100644 --- a/lib/umbrellio_utils/vault.rb +++ b/lib/umbrellio_utils/vault.rb @@ -22,7 +22,7 @@ def create_kv_engine(path) def write_to_kv(engine_path:, secret_path:, data:) full_data_path = File.join(engine_path, "data", secret_path) full_meta_path = File.join(engine_path, "metadata", secret_path) - ::Vault.logical.write(full_data_path, data: data) + ::Vault.logical.write(full_data_path, data:) ::Vault.logical.write(full_meta_path, id: secret_path, max_versions: 1, cas_required: false) end end diff --git a/lib/umbrellio_utils/version.rb b/lib/umbrellio_utils/version.rb index 5d058d2..70633d7 100644 --- a/lib/umbrellio_utils/version.rb +++ b/lib/umbrellio_utils/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module UmbrellioUtils - VERSION = "1.4.0" + VERSION = "1.5.0" end diff --git a/spec/support/sequel_patches.rb b/spec/support/sequel_patches.rb index de1c098..45a90e3 100644 --- a/spec/support/sequel_patches.rb +++ b/spec/support/sequel_patches.rb @@ -8,7 +8,7 @@ def select_sql return super if @opts[:_skip_order_patch] order = @opts[:order].dup || [] order << Sequel.function(:random) - clone(order: order, _skip_order_patch: true).select_sql + clone(order:, _skip_order_patch: true).select_sql end end end diff --git a/spec/umbrellio_utils/jobs_spec.rb b/spec/umbrellio_utils/jobs_spec.rb new file mode 100644 index 0000000..c30ea87 --- /dev/null +++ b/spec/umbrellio_utils/jobs_spec.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +describe UmbrellioUtils::Jobs do + let(:jobs) do + Module.new do + extend UmbrellioUtils::Jobs + end + end + + before do + jobs.workers.clear + jobs.capsules.clear + jobs.queues.clear + + jobs.register_worker(:default) + jobs.register_worker(:w1) + + jobs.register_capsule(:default) + jobs.register_capsule(:cap1, weight: 5) + jobs.register_capsule(:cap2, weight: 10) + jobs.register_capsule(:cap3, worker: :w1) + jobs.register_capsule(:cap4) + + jobs.register_queue(:q1, capsule: :cap1) + jobs.register_queue(:q2, capsule: :cap2, weight: 5) + jobs.register_queue(:q3, weight: 10) + jobs.register_queue(:q4) + jobs.register_queue(:q5, capsule: :cap3) + end + + describe ".capsules_for" do + let(:level) { "default" } + let(:max_concurrency) { 10 } + let(:result) { jobs.capsules_for(level, max_concurrency) } + + specify do + expect(result).to eq( + [ + UmbrellioUtils::Jobs::Entry.new(:default, [[:q3, 10], [:q4, 1]], 1), + UmbrellioUtils::Jobs::Entry.new(:cap1, [[:q1, 1]], 3), + UmbrellioUtils::Jobs::Entry.new(:cap2, [[:q2, 5]], 6), + ], + ) + end + + context "non default level" do + let(:level) { "w1" } + + specify do + expect(result).to eq( + [ + UmbrellioUtils::Jobs::Entry.new(:cap3, [[:q5, 1]], 10), + ], + ) + end + end + + context "non existent level" do + let(:level) { "invalid" } + + specify do + expect { result }.to raise_error('No queues found for worker "invalid"') + end + end + end + + describe ".configure_capsules!" do + before do + allow(config).to receive(:capsule).with(:default).and_yield(capsule_default) + allow(config).to receive(:capsule).with(:cap1).and_yield(capsule_cap1) + allow(config).to receive(:capsule).with(:cap2).and_yield(capsule_cap2) + end + + let(:capsule) { Struct.new(:queues, :concurrency) } + let(:config) { double(:config) } + let(:capsule_default) { capsule.new } + let(:capsule_cap1) { capsule.new } # rubocop:disable RSpec/IndexedLet + let(:capsule_cap2) { capsule.new } # rubocop:disable RSpec/IndexedLet + + specify do + jobs.configure_capsules!(config, priority_level: "default", max_concurrency: 10) + + expect(capsule_default.queues).to eq([[:q3, 10], [:q4, 1]]) + expect(capsule_default.concurrency).to eq(1) + + expect(capsule_cap1.queues).to eq([[:q1, 1]]) + expect(capsule_cap1.concurrency).to eq(3) + + expect(capsule_cap2.queues).to eq([[:q2, 5]]) + expect(capsule_cap2.concurrency).to eq(6) + end + + context "non default level" do + specify do + jobs.configure_capsules!(config, priority_level: "w1", max_concurrency: 10) + + expect(capsule_default.queues).to eq([[:q5, 1]]) + expect(capsule_default.concurrency).to eq(10) + + expect(capsule_cap1.queues).to eq(nil) + expect(capsule_cap2.queues).to eq(nil) + end + end + end + + describe ".validate_queue_name!" do + specify do + expect(jobs.validate_queue_name!(:q1)).to eq(nil) + end + + specify do + expect { jobs.validate_queue_name!(:invalid) }.to raise_error("Unknown queue: :invalid") + end + end + + describe ".retry_interval" do + specify do + expect(jobs.retry_interval(1, min_interval: 10, max_interval: 100)).to eq(10) + expect(jobs.retry_interval(5, min_interval: 10, max_interval: 100)).to eq(17) + expect(jobs.retry_interval(10, min_interval: 10, max_interval: 100)).to eq(63) + end + end +end diff --git a/spec/umbrellio_utils/request_wrapper_spec.rb b/spec/umbrellio_utils/request_wrapper_spec.rb index b8e5640..d0a8750 100644 --- a/spec/umbrellio_utils/request_wrapper_spec.rb +++ b/spec/umbrellio_utils/request_wrapper_spec.rb @@ -2,7 +2,7 @@ describe UmbrellioUtils::RequestWrapper do subject(:wrapped_request) do - described_class.new(request, remove_xml_attributes: remove_xml_attributes) + described_class.new(request, remove_xml_attributes:) end let(:request_body) { Hash[some: "value"].to_json } diff --git a/spec/umbrellio_utils/semantic_logger/tiny_json_formatter_spec.rb b/spec/umbrellio_utils/semantic_logger/tiny_json_formatter_spec.rb index 4053b4a..3315033 100644 --- a/spec/umbrellio_utils/semantic_logger/tiny_json_formatter_spec.rb +++ b/spec/umbrellio_utils/semantic_logger/tiny_json_formatter_spec.rb @@ -30,7 +30,7 @@ let(:result) { formatter.call(log, nil) } let(:formatter) { described_class.new(**options) } - let(:options) { Hash[custom_names_mapping: custom_names_mapping] } + let(:options) { Hash[custom_names_mapping:] } let(:custom_names_mapping) { Hash[] } let(:log_level) { :debug } diff --git a/umbrellio_utils.gemspec b/umbrellio_utils.gemspec index 1381927..f9566d6 100644 --- a/umbrellio_utils.gemspec +++ b/umbrellio_utils.gemspec @@ -12,7 +12,7 @@ Gem::Specification.new do |spec| spec.description = "UmbrellioUtils is collection of utility classes and helpers" spec.homepage = "https://github.com/umbrellio/utils" spec.license = "MIT" - spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0") + spec.required_ruby_version = Gem::Requirement.new(">= 3.1.0") spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = "https://github.com/umbrellio/utils"