diff --git a/.github/workflows/judoscale-solid_queue.yml b/.github/workflows/judoscale-solid_queue.yml new file mode 100644 index 00000000..b9cfb451 --- /dev/null +++ b/.github/workflows/judoscale-solid_queue.yml @@ -0,0 +1,45 @@ +name: judoscale-solid_queue tests +defaults: + run: + working-directory: judoscale-solid_queue +on: + push: + branches: + - main + pull_request: +jobs: + test: + strategy: + fail-fast: false + matrix: + gemfile: + - Gemfile + ruby: + - "2.7" + - "3.0" + - "3.1" + - "3.2" + - "3.3" + exclude: + + runs-on: ubuntu-latest + env: # $BUNDLE_GEMFILE must be set at the job level, so it is set for all steps + BUNDLE_GEMFILE: ${{ github.workspace }}/judoscale-solid_queue/${{ matrix.gemfile }} + services: + db: + image: postgres:latest + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true # runs bundle install and caches installed gems automatically + - run: bundle exec rake diff --git a/judoscale-good_job/test/test_helper.rb b/judoscale-good_job/test/test_helper.rb index cfafc21e..7ec067d4 100644 --- a/judoscale-good_job/test/test_helper.rb +++ b/judoscale-good_job/test/test_helper.rb @@ -10,6 +10,7 @@ require "action_controller" class TestRailsApp < Rails::Application + config.load_defaults "#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}" config.secret_key_base = "test-secret" config.eager_load = false config.logger = ::Logger.new(StringIO.new, progname: "rails-app") diff --git a/judoscale-rails/test/test_helper.rb b/judoscale-rails/test/test_helper.rb index 28eb7bbb..9ff2e4f7 100644 --- a/judoscale-rails/test/test_helper.rb +++ b/judoscale-rails/test/test_helper.rb @@ -11,6 +11,7 @@ require "action_controller" class TestRailsApp < Rails::Application + config.load_defaults "#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}" config.secret_key_base = "test-secret" config.eager_load = false config.logger = ::Logger.new(StringIO.new, progname: "rails-app") diff --git a/judoscale-solid_queue/Gemfile b/judoscale-solid_queue/Gemfile new file mode 100644 index 00000000..5a989d6b --- /dev/null +++ b/judoscale-solid_queue/Gemfile @@ -0,0 +1,9 @@ +source "https://rubygems.org" + +gemspec name: "judoscale-solid_queue" + +gem "judoscale-ruby", path: "../judoscale-ruby" +gem "activerecord" +gem "pg" +gem "minitest" +gem "rake" diff --git a/judoscale-solid_queue/Rakefile b/judoscale-solid_queue/Rakefile new file mode 100644 index 00000000..b4a24c8f --- /dev/null +++ b/judoscale-solid_queue/Rakefile @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "rake/testtask" + +Rake::TestTask.new(:test) do |t| + t.libs << "lib" + t.libs << "test" + t.test_files = FileList["test/**/*_test.rb"] +end + +task default: :test diff --git a/judoscale-solid_queue/judoscale-solid_queue.gemspec b/judoscale-solid_queue/judoscale-solid_queue.gemspec new file mode 100644 index 00000000..02b60676 --- /dev/null +++ b/judoscale-solid_queue/judoscale-solid_queue.gemspec @@ -0,0 +1,30 @@ +lib = File.expand_path("../lib", __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require "judoscale/solid_queue/version" + +Gem::Specification.new do |spec| + spec.name = "judoscale-solid_queue" + spec.version = Judoscale::SolidQueue::VERSION + spec.authors = ["Adam McCrea", "Carlos Antonio da Silva", "Jon Sullivan"] + spec.email = ["hello@judoscale.com"] + + spec.summary = "This gem provides SolidQueue integration with the Judoscale autoscaling add-on for Heroku." + spec.homepage = "https://judoscale.com" + spec.license = "MIT" + + spec.metadata = { + "homepage_uri" => "https://judoscale.com", + "bug_tracker_uri" => "https://github.com/judoscale/judoscale-ruby/issues", + "documentation_uri" => "https://judoscale.com/docs", + "changelog_uri" => "https://github.com/judoscale/judoscale-ruby/blob/main/CHANGELOG.md", + "source_code_uri" => "https://github.com/judoscale/judoscale-ruby" + } + + spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + spec.require_paths = ["lib"] + + spec.required_ruby_version = ">= 2.7.0" + + spec.add_dependency "judoscale-ruby", Judoscale::SolidQueue::VERSION + spec.add_dependency "solid_queue", ">= 0.3" +end diff --git a/judoscale-solid_queue/lib/judoscale-solid_queue.rb b/judoscale-solid_queue/lib/judoscale-solid_queue.rb new file mode 100644 index 00000000..bebf4cfb --- /dev/null +++ b/judoscale-solid_queue/lib/judoscale-solid_queue.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require "judoscale/solid_queue" diff --git a/judoscale-solid_queue/lib/judoscale/solid_queue.rb b/judoscale-solid_queue/lib/judoscale/solid_queue.rb new file mode 100644 index 00000000..2bc50f7c --- /dev/null +++ b/judoscale-solid_queue/lib/judoscale/solid_queue.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require "solid_queue" +require "judoscale-ruby" +require "judoscale/config" +require "judoscale/solid_queue/version" +require "judoscale/solid_queue/metrics_collector" + +Judoscale.add_adapter :"judoscale-solid_queue", + { + adapter_version: Judoscale::SolidQueue::VERSION, + framework_version: ::SolidQueue::VERSION + }, + metrics_collector: Judoscale::SolidQueue::MetricsCollector, + expose_config: Judoscale::Config::JobAdapterConfig.new(:solid_queue) diff --git a/judoscale-solid_queue/lib/judoscale/solid_queue/metrics_collector.rb b/judoscale-solid_queue/lib/judoscale/solid_queue/metrics_collector.rb new file mode 100644 index 00000000..0cc27bf3 --- /dev/null +++ b/judoscale-solid_queue/lib/judoscale/solid_queue/metrics_collector.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require "judoscale/job_metrics_collector" +require "judoscale/job_metrics_collector/active_record_helper" +require "judoscale/metric" + +module Judoscale + module SolidQueue + class MetricsCollector < Judoscale::JobMetricsCollector + include ActiveRecordHelper + + def self.adapter_config + Judoscale::Config.instance.solid_queue + end + + def self.collect?(config) + super && ActiveRecordHelper.table_exists_for_model?(::SolidQueue::Job) + end + + def initialize + super + + queue_names = run_silently do + ::SolidQueue::Job.distinct.pluck(:queue_name) + end + self.queues |= queue_names + end + + def collect + metrics = [] + time = Time.now.utc + + oldest_execution_time_by_queue = run_silently do + ::SolidQueue::ReadyExecution + .group(:queue_name) + .minimum(:created_at) + end + self.queues |= oldest_execution_time_by_queue.keys + + if track_busy_jobs? + busy_count_by_queue = run_silently do + ::SolidQueue::Job.joins(:claimed_execution).group(:queue_name).count + end + self.queues |= busy_count_by_queue.keys + end + + queues.each do |queue| + run_at = oldest_execution_time_by_queue[queue] + latency_ms = run_at ? ((time - run_at) * 1000).ceil : 0 + latency_ms = 0 if latency_ms < 0 + + metrics.push Metric.new(:qt, latency_ms, time, queue) + + if track_busy_jobs? + busy_count = busy_count_by_queue[queue] || 0 + metrics.push Metric.new(:busy, busy_count, Time.now, queue) + end + end + + log_collection(metrics) + metrics + end + end + end +end diff --git a/judoscale-solid_queue/lib/judoscale/solid_queue/version.rb b/judoscale-solid_queue/lib/judoscale/solid_queue/version.rb new file mode 100644 index 00000000..5fdb4e5f --- /dev/null +++ b/judoscale-solid_queue/lib/judoscale/solid_queue/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Judoscale + module SolidQueue + VERSION = "1.5.4" + end +end diff --git a/judoscale-solid_queue/lib/rails-autoscale-solid_queue.rb b/judoscale-solid_queue/lib/rails-autoscale-solid_queue.rb new file mode 100644 index 00000000..bebf4cfb --- /dev/null +++ b/judoscale-solid_queue/lib/rails-autoscale-solid_queue.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require "judoscale/solid_queue" diff --git a/judoscale-solid_queue/rails-autoscale-solid_queue.gemspec b/judoscale-solid_queue/rails-autoscale-solid_queue.gemspec new file mode 100644 index 00000000..e52f01b5 --- /dev/null +++ b/judoscale-solid_queue/rails-autoscale-solid_queue.gemspec @@ -0,0 +1,30 @@ +lib = File.expand_path("../lib", __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require "judoscale/solid_queue/version" + +Gem::Specification.new do |spec| + spec.name = "rails-autoscale-solid_queue" + spec.version = Judoscale::SolidQueue::VERSION + spec.authors = ["Adam McCrea", "Carlos Antonio da Silva", "Jon Sullivan"] + spec.email = ["hello@judoscale.com"] + + spec.summary = "This gem provides SolidQueue integration with the Judoscale autoscaling add-on for Heroku." + spec.homepage = "https://judoscale.com" + spec.license = "MIT" + + spec.metadata = { + "homepage_uri" => "https://judoscale.com", + "bug_tracker_uri" => "https://github.com/judoscale/judoscale-ruby/issues", + "documentation_uri" => "https://judoscale.com/docs", + "changelog_uri" => "https://github.com/judoscale/judoscale-ruby/blob/main/CHANGELOG.md", + "source_code_uri" => "https://github.com/judoscale/judoscale-ruby" + } + + spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + spec.require_paths = ["lib"] + + spec.required_ruby_version = ">= 2.6.0" + + spec.add_dependency "rails-autoscale-core", Judoscale::SolidQueue::VERSION + spec.add_dependency "solid_queue", ">= 0.3" +end diff --git a/judoscale-solid_queue/test/adapter_test.rb b/judoscale-solid_queue/test/adapter_test.rb new file mode 100644 index 00000000..8bfadaa0 --- /dev/null +++ b/judoscale-solid_queue/test/adapter_test.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require "test_helper" +require "judoscale/report" + +module Judoscale + describe SolidQueue do + it "adds itself as an adapter with information to be reported to the Judoscale API" do + adapter = Judoscale.adapters.detect { |adapter| adapter.identifier == :"judoscale-solid_queue" } + _(adapter).wont_be_nil + _(adapter.metrics_collector).must_equal Judoscale::SolidQueue::MetricsCollector + + report = ::Judoscale::Report.new(Judoscale.adapters, Judoscale::Config.instance, []) + _(report.as_json[:adapters]).must_include(:"judoscale-solid_queue") + end + + it "sets up a config property for the library" do + config = Config.instance + _(config.solid_queue.enabled).must_equal true + _(config.solid_queue.max_queues).must_equal 20 + _(config.solid_queue.queues).must_equal [] + _(config.solid_queue.track_busy_jobs).must_equal false + + Judoscale.configure do |config| + config.solid_queue.queues = %w[test drive] + config.solid_queue.track_busy_jobs = true + end + + _(config.solid_queue.queues).must_equal %w[test drive] + _(config.solid_queue.track_busy_jobs).must_equal true + + report = ::Judoscale::Report.new(Judoscale.adapters, Judoscale::Config.instance, []) + _(report.as_json[:config]).must_include(:solid_queue) + end + end +end diff --git a/judoscale-solid_queue/test/metrics_collector_test.rb b/judoscale-solid_queue/test/metrics_collector_test.rb new file mode 100644 index 00000000..9dabb36a --- /dev/null +++ b/judoscale-solid_queue/test/metrics_collector_test.rb @@ -0,0 +1,258 @@ +# frozen_string_literal: true + +require "test_helper" +require "judoscale/solid_queue/metrics_collector" + +class DelayableWithoutRetry < ActiveJob::Base + def perform(succeed = true) + raise "boom" unless succeed + end +end + +class Delayable < DelayableWithoutRetry + retry_on StandardError +end + +module Judoscale + describe SolidQueue::MetricsCollector do + subject { SolidQueue::MetricsCollector.new } + + def clear_enqueued_jobs + ::SolidQueue::Job.delete_all + ::SolidQueue::ReadyExecution.delete_all + end + + describe "#collect" do + after { + clear_enqueued_jobs + subject.clear_queues + } + + it "collects latency for each queue, using the oldest enqueued job" do + now = Time.now.utc + + freeze_time now - 0.15 do + Delayable.set(queue: "default").perform_later + end + + metrics = freeze_time now do + Delayable.set(queue: "default").perform_later + Delayable.set(queue: "high").perform_later + + subject.collect + end + + _(metrics.size).must_equal 2 + _(metrics.map(&:queue_name).sort).must_equal %w[default high] + + metrics_hash = metrics.map { |m| [m.queue_name, m] }.to_h + + _(metrics_hash["default"].queue_name).must_equal "default" + _(metrics_hash["default"].value).must_be_within_delta 150, 1 + _(metrics_hash["default"].identifier).must_equal :qt + _(metrics_hash["high"].queue_name).must_equal "high" + _(metrics_hash["high"].value).must_be_within_delta 0, 1 + _(metrics_hash["high"].identifier).must_equal :qt + end + + it "always collects for known queues" do + Delayable.set(queue: "high").perform_later + + metrics = subject.collect + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "high" + + Delayable.set(queue: "default").perform_later + + metrics = subject.collect + + _(metrics.map(&:queue_name).sort).must_equal %w[default high] + + clear_enqueued_jobs + metrics = subject.collect + + _(metrics.size).must_equal 2 + _(metrics.map(&:queue_name).sort).must_equal %w[default high] + end + + it "always collects for queues with completed jobs" do + metrics = subject.collect + + _(metrics).must_be :empty? + + now = Time.now.utc + freeze_time(now - 0.15) { Delayable.set(queue: "default").perform_later } + metrics = freeze_time(now) { subject.collect } + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "default" + _(metrics[0].value).must_be_within_delta 150, 1 + + ::SolidQueue::ReadyExecution.claim(%w[default], 1, 42).each(&:perform) + _(::SolidQueue::Job.finished.count).must_equal 1 + + metrics = subject.collect + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "default" + _(metrics[0].value).must_equal 0 + end + + it "ignores future jobs" do + Delayable.set(queue: "default", wait: 10.seconds).perform_later + + metrics = subject.collect + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "default" + _(metrics[0].value).must_equal 0 + end + + it "ignores claimed jobs being processed" do + freeze_time Time.now - 1 do + Delayable.set(queue: "default").perform_later + end + + ::SolidQueue::ReadyExecution.claim(%w[default], 1, 42) + _(::SolidQueue::ClaimedExecution.count).must_equal 1 + + # require 'debug';debugger + metrics = subject.collect + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "default" + _(metrics[0].value).must_equal 0 + end + + it "ignores failed jobs waiting on retry (re-scheduled via Active Job)" do + freeze_time Time.now - 1 do + Delayable.set(queue: "default").perform_later(false) + end + + ::SolidQueue::ReadyExecution.claim(%w[default], 1, 42).each(&:perform) + _(::SolidQueue::ScheduledExecution.count).must_equal 1 + + metrics = subject.collect + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "default" + _(metrics[0].value).must_equal 0 + end + + it "ignores failed jobs" do + freeze_time Time.now - 1 do + DelayableWithoutRetry.set(queue: "default").perform_later(false) + end + + ::SolidQueue::ReadyExecution.claim(%w[default], 1, 42).each(&:perform) + _(::SolidQueue::FailedExecution.count).must_equal 1 + + metrics = subject.collect + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "default" + _(metrics[0].value).must_equal 0 + end + + it "collects metrics for jobs without a queue name" do + metrics = freeze_time do + Delayable.perform_later + + subject.collect + end + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "default" + _(metrics[0].value).must_be_within_delta 0, 1 + end + + it "logs debug information for each queue being collected" do + use_config log_level: :debug do + Delayable.set(queue: "default").perform_later + + subject.collect + + _(log_string).must_match %r{solid_queue-qt.default=\d+ms} + _(log_string).wont_match %r{solid_queue-busy} + end + end + + it "tracks busy jobs when the configuration is enabled" do + use_adapter_config :solid_queue, track_busy_jobs: true do + Delayable.set(queue: "default").perform_later + + ::SolidQueue::ReadyExecution.claim(%w[default], 1, 42) + + metrics = subject.collect + + _(metrics.size).must_equal 2 + _(metrics[1].value).must_equal 1 + _(metrics[1].queue_name).must_equal "default" + _(metrics[1].identifier).must_equal :busy + end + end + + it "logs debug information about busy jobs being collected" do + use_config log_level: :debug do + use_adapter_config :solid_queue, track_busy_jobs: true do + Delayable.set(queue: "default").perform_later + + ::SolidQueue::ReadyExecution.claim(%w[default], 1, 42) + + subject.collect + + _(log_string).must_match %r{solid_queue-qt.default=.+ solid_queue-busy.default=1} + end + end + end + + it "filters queues matching UUID format by default, to prevent reporting for dynamically generated queues" do + %W[low-#{SecureRandom.uuid} default #{SecureRandom.uuid}-high].each { |queue| + Delayable.set(queue: queue).perform_later + } + + metrics = subject.collect + + _(metrics.size).must_equal 1 + _(metrics[0].queue_name).must_equal "default" + end + + it "filters queues to collect metrics from based on the configured queue filter proc, overriding the default UUID filter" do + use_adapter_config :solid_queue, queue_filter: ->(queue_name) { queue_name.start_with? "low" } do + %W[low default high low-#{SecureRandom.uuid}].each { |queue| + Delayable.set(queue: queue).perform_later + } + + metrics = subject.collect + + queue_names = metrics.map(&:queue_name).sort + _(queue_names.size).must_equal 2 + _(queue_names[0]).must_equal "low" + _(queue_names[1]).must_be :start_with?, "low-" + end + end + + it "collects metrics only from the configured queues if the configuration is present, ignoring the queue filter" do + use_adapter_config :solid_queue, queues: %w[low ultra], queue_filter: ->(queue_name) { queue_name != "low" } do + %w[low default high].each { |queue| Delayable.set(queue: queue).perform_later } + + metrics = subject.collect + + _(metrics.map(&:queue_name)).must_equal %w[low ultra] + end + end + + it "collects metrics up to the configured number of max queues, sorting by length of the queue name" do + use_adapter_config :solid_queue, max_queues: 2 do + %w[low default high].each { |queue| Delayable.set(queue: queue).perform_later } + + metrics = subject.collect + + _(metrics.map(&:queue_name)).must_equal %w[low high] + _(log_string).must_match %r{SolidQueue metrics reporting only 2 queues max, skipping the rest \(1\)} + end + end + end + end +end diff --git a/judoscale-solid_queue/test/test_helper.rb b/judoscale-solid_queue/test/test_helper.rb new file mode 100644 index 00000000..d74dd685 --- /dev/null +++ b/judoscale-solid_queue/test/test_helper.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) + +# Stuff we need to require for SolidQueue +require "rails" +require "active_job/railtie" +# SolidQueue fixed these requires on main, they can be removed eventually +# https://github.com/rails/solid_queue/commit/5ff6e0178bbe7c0cf93134ea2ef974c1dfc09a09 +require "active_support" +require "active_support/core_ext/numeric/time" +require "judoscale-solid_queue" + +require "minitest/autorun" +require "minitest/spec" + +ENV["RACK_ENV"] ||= "test" +require "action_controller" + +class TestRailsApp < Rails::Application + config.load_defaults "#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}" + config.secret_key_base = "test-secret" + config.eager_load = false + config.logger = ::Logger.new(StringIO.new, progname: "rails-app") + config.active_job.queue_adapter = :solid_queue + routes.append do + root to: proc { + [200, {"Content-Type" => "text/plain"}, ["Hello World"]] + } + end + initialize! +end + +require "active_record" + +DATABASE_NAME = "judoscale_solid_queue_test" +DATABASE_USERNAME = "postgres" +DATABASE_URL = "postgres://#{DATABASE_USERNAME}:@localhost/#{DATABASE_NAME}" + +ActiveRecord::Tasks::DatabaseTasks.create(DATABASE_URL) +Minitest.after_run { + ActiveRecord::Tasks::DatabaseTasks.drop(DATABASE_URL) +} +ActiveRecord::Base.establish_connection(DATABASE_URL) + +# Suppress migration noise. +ENV["VERBOSE"] = "false" +# Add SolidQueue migration path to Active Record to migrate to the latest automatically. +# It seems we can't only set it on `DatabaseTasks` as expected, need to set on the `Migrator` directly instead. +ActiveRecord::Migrator.migrations_paths += SolidQueue::Engine.config.paths["db/migrate"].existent +# ActiveRecord::Tasks::DatabaseTasks.migrations_paths += SolidQueue::Engine.config.paths["db/migrate"].existent +ActiveRecord::Tasks::DatabaseTasks.migrate + +module Judoscale::Test +end + +Dir[File.expand_path("../../judoscale-ruby/test/support/*.rb", __dir__)].sort.each { |file| require file } + +Minitest::Test.include(Judoscale::Test)