diff --git a/judoscale-rails/lib/judoscale/rails/railtie.rb b/judoscale-rails/lib/judoscale/rails/railtie.rb
index 0d8c5fc8..33ec3731 100644
--- a/judoscale-rails/lib/judoscale/rails/railtie.rb
+++ b/judoscale-rails/lib/judoscale/rails/railtie.rb
@@ -17,7 +17,7 @@ def in_rails_console?
       end
 
       def in_rake_task?
-        defined?(::Rake) && Rake.respond_to?(:application)
+        defined?(::Rake) && Rake.respond_to?(:application) && Rake.application.top_level_tasks.any?
       end
 
       initializer "Judoscale.logger" do |app|
diff --git a/judoscale-ruby/lib/judoscale/metrics_collector.rb b/judoscale-ruby/lib/judoscale/metrics_collector.rb
index de048453..48d754ff 100644
--- a/judoscale-ruby/lib/judoscale/metrics_collector.rb
+++ b/judoscale-ruby/lib/judoscale/metrics_collector.rb
@@ -3,7 +3,7 @@
 module Judoscale
   class MetricsCollector
     def self.collect?(config)
-      in_rake_task = defined?(::Rake) && Rake.respond_to?(:application)
+      in_rake_task = defined?(::Rake) && Rake.respond_to?(:application) && Rake.application.top_level_tasks.any?
 
       !in_rake_task || in_whitelisted_rake_tasks?(config.allow_rake_tasks)
     end
diff --git a/judoscale-ruby/test/job_metrics_collector_test.rb b/judoscale-ruby/test/job_metrics_collector_test.rb
index 855a2e3d..55fea6a4 100644
--- a/judoscale-ruby/test/job_metrics_collector_test.rb
+++ b/judoscale-ruby/test/job_metrics_collector_test.rb
@@ -12,10 +12,14 @@ module Judoscale
         Object.stub_const :Rake, nil do
           _(WebMetricsCollector.collect?(Config.instance)).must_equal true
         end
+
+        Object.stub_const :Rake, RakeMock.new([]) do
+          _(WebMetricsCollector.collect?(Config.instance)).must_equal true
+        end
       end
 
       it "returns false when running in a rake task" do
-        Object.stub_const :Rake, RakeMock.new do
+        Object.stub_const :Rake, RakeMock.new(["foo"]) do
           _(WebMetricsCollector.collect?(Config.instance)).must_equal false
         end
       end
diff --git a/judoscale-ruby/test/rake_mock.rb b/judoscale-ruby/test/rake_mock.rb
index 52ac4ebb..2f0acba8 100644
--- a/judoscale-ruby/test/rake_mock.rb
+++ b/judoscale-ruby/test/rake_mock.rb
@@ -1,5 +1,5 @@
 class RakeMock
-  def initialize(top_level_tasks = [])
+  def initialize(top_level_tasks)
     @top_level_tasks = top_level_tasks
   end
 
diff --git a/judoscale-ruby/test/web_metrics_collector_test.rb b/judoscale-ruby/test/web_metrics_collector_test.rb
index 55504a58..47ddb322 100644
--- a/judoscale-ruby/test/web_metrics_collector_test.rb
+++ b/judoscale-ruby/test/web_metrics_collector_test.rb
@@ -13,10 +13,14 @@ module Judoscale
         Object.stub_const :Rake, nil do
           _(WebMetricsCollector.collect?(Config.instance)).must_equal true
         end
+
+        Object.stub_const :Rake, RakeMock.new([]) do
+          _(WebMetricsCollector.collect?(Config.instance)).must_equal true
+        end
       end
 
       it "returns false when running in a rake task" do
-        Object.stub_const :Rake, RakeMock.new do
+        Object.stub_const :Rake, RakeMock.new(["foo"]) do
           _(WebMetricsCollector.collect?(Config.instance)).must_equal false
         end
       end
diff --git a/sample-apps/rails-sample/Gemfile.lock b/sample-apps/rails-sample/Gemfile.lock
index fad87782..7a51ea77 100644
--- a/sample-apps/rails-sample/Gemfile.lock
+++ b/sample-apps/rails-sample/Gemfile.lock
@@ -1,14 +1,14 @@
 PATH
   remote: ../../judoscale-rails
   specs:
-    judoscale-rails (1.3.0)
-      judoscale-ruby (= 1.3.0)
+    judoscale-rails (1.5.0)
+      judoscale-ruby (= 1.5.0)
       railties
 
 PATH
   remote: ../../judoscale-ruby
   specs:
-    judoscale-ruby (1.3.0)
+    judoscale-ruby (1.5.0)
 
 GEM
   remote: https://rubygems.org/
@@ -76,6 +76,7 @@ GEM
 PLATFORMS
   arm64-darwin-20
   arm64-darwin-21
+  arm64-darwin-22
   x86_64-darwin-21
   x86_64-linux