Add benchmark hey files and md description [ci skip] (puma#3564)

* Add benchmark hey files and md description

* CI: benchmarks/local/long_tail_hey.md - fix typos

* CI: benchmarks/local/long_tail_hey.md - add YJIT option, format tables

benchmarks/local/bench_base.rb

@@ -56,7 +56,7 @@ class BenchBase
     SIZES_RE = /\d[\d,]*\z/.freeze
 
     def initialize
-      sleep 5 # wait for server to boot
+      sleep 0.2     # wait for server to boot
 
       @thread_loops       = nil
       @clients_per_thread = nil
@@ -66,6 +66,9 @@ def initialize
       @dly_app            = nil
       @bind_type          = :tcp
 
+      @worker_req_ttl = {}
+      @pids = {}
+
       @ios_to_close = []
 
       setup_options
@@ -101,35 +104,25 @@ def initialize
     end
 
     def setup_options
+      # STDOUT.syswrite "\n\n#{ARGV}\n"
+
       OptionParser.new do |o|
         o.on "-T", "--stream-threads THREADS", OptionParser::DecimalInteger, "request_stream: loops/threads" do |arg|
           @stream_threads = arg.to_i
         end
 
         o.on "-c", "--wrk-connections CONNECTIONS", OptionParser::DecimalInteger, "request_stream: clients_per_thread" do |arg|
-          @wrk_connections = arg.to_i
+          @connections = arg.to_i
         end
 
         o.on "-R", "--requests REQUESTS", OptionParser::DecimalInteger, "request_stream: requests per socket" do |arg|
-          @req_per_socket = arg.to_i
+          @req_per_connection = arg.to_i
         end
 
         o.on "-D", "--duration DURATION", OptionParser::DecimalInteger, "wrk/stream: duration" do |arg|
           @duration = arg.to_i
         end
 
-        o.on "-b", "--body_conf BODY_CONF", String, "CI RackUp: type and size of response body in kB" do |arg|
-          if (types = arg[TYPES_RE])
-            @body_types = TYPES.select { |a| types.include? a[0].to_s }
-          end
-
-          if (sizes = arg[SIZES_RE])
-            @body_sizes = sizes.split(',')
-            @body_sizes.map!(&:to_i)
-            @body_sizes.sort!
-          end
-        end
-
         o.on "-d", "--dly_app DELAYAPP", Float, "CI RackUp: app response delay" do |arg|
           @dly_app = arg.to_f
         end
@@ -143,7 +136,7 @@ def setup_options
         end
 
         o.on "-t", "--threads PUMA_THREADS", String, "Puma Server: threads" do |arg|
-          @threads = arg
+          @threads =  arg.match?(/\d+:\d+/) ? arg[/\d+\z/].to_i : arg.to_i
         end
 
         o.on "-w", "--workers PUMA_WORKERS", OptionParser::DecimalInteger, "Puma Server: workers" do |arg|
@@ -154,6 +147,10 @@ def setup_options
           @wrk_bind_str = arg
         end
 
+        o.on "-k", "--disable-keepalive", "hey no keep alive" do
+          @no_keep_alive = true
+        end
+
         o.on("-h", "--help", "Prints this help") do
           puts o
           exit
@@ -179,6 +176,44 @@ def close_clients
       puts "Closed #{closed} sockets" unless closed.zero?
     end
 
+    # Runs hey and returns data from its output.
+    # @param cmd [String] The hey command string, with arguments
+    # @return [Hash] The hey data
+    #
+    def run_hey_parse(cmd, mult, log: false)
+      STDOUT.syswrite "#{cmd}\n"
+
+      hey_output = %x[#{cmd}].strip.gsub(' secs', '')
+
+      if log
+        puts '', hey_output, ''
+      end
+
+      job = {}
+
+      status_code_dist = hey_output[/^Status code distribution:\s+(.+?)\z/m, 1].strip.gsub(/^\s+\[/, '[')
+
+      if status_code_dist.include? "\n"
+        job[:error] = status_code_dist
+        STDOUT.syswrite "\nERRORS:\n#{status_code_dist}\n"
+      end
+
+      job[:mult] = format '%1.2f', mult
+
+      job[:rps] = hey_output[/^\s+Requests\/sec\:\s+([\d.]+)/, 1]
+
+      latency = hey_output[/^Latency distribution:\n(.+?)\n\n/m, 1].gsub(/^ +/, '').gsub('in ', '').gsub(' secs', '')
+      if latency
+        temp = job[:latency] = {}
+        latency.lines.each do |l|
+          per_cent = l[/\A\d+/].to_i
+          temp[per_cent] = per_cent.zero? ? '  na' : l.rstrip[/[\d.]+\z/]
+        end
+        temp[100] = hey_output[/^\s+Slowest\:\s+([\d.]+)/, 1]
+      end
+      job
+    end
+
     # Runs wrk and returns data from its output.
     # @param cmd [String] The wrk command string, with arguments
     # @return [Hash] The wrk data
@@ -297,18 +332,17 @@ def env_log
       end
     end
 
-    # Parses data returned by `PumaInfo.run stats`
+    # Parses data returned by `PumaInfo.run 'stats'`
     # @return [Hash] The data from Puma stats
     #
     def parse_stats
-      stats = {}
-
+      sleep 5.0
       obj = @puma_info.run 'stats'
-
-      worker_status = obj[:worker_status]
-
+      return nil unless worker_status = obj[:worker_status]
+      stats = {}
       worker_status.each do |w|
         pid = w[:pid]
+        @worker_req_ttl[pid] ||= 0
         req_cnt = w[:last_status][:requests_count]
         id = format 'worker-%01d-%02d', w[:phase], w[:index]
         hsh = {

benchmarks/local/bench_base.sh

@@ -33,7 +33,7 @@ if [ -z "$PUMA_TEST_STATE" ]; then export PUMA_TEST_STATE=tmp/bench_test_puma.st
 
 export PUMA_CTRL=$PUMA_TEST_HOST4:$PUMA_TEST_CTRL
 
-while getopts :b:C:c:D:d:R:r:s:T:t:w:Y option
+while getopts :b:C:c:D:d:kR:r:s:T:t:w:Y option
 do
 case "${option}" in
 #———————————————————— RUBY options
@@ -55,6 +55,8 @@ R) req_per_socket=${OPTARG};;
 c) connections=${OPTARG};;
 # T) stream_threads=${OPTARG};;
 # D) duration=${OPTARG};;
+#———————————————————— hey options
+k) disable_keepalive=true;;
 ?) echo "Error: Invalid option was specified -$OPTARG"; exit;;
 esac
 done
@@ -101,6 +103,11 @@ if [ -n "$workers" ]; then
   ruby_args="$ruby_args -w$workers"
 fi
 
+
+if [ -n "$disable_keepalive" ]; then
+  ruby_args="$ruby_args -k"
+fi
+
 if [ -z "$threads" ]; then
   threads=0:5
 fi
@@ -113,7 +120,7 @@ if [ -n "$conf" ]; then
 fi
 
 if [ -z "$rackup_file" ]; then
-  rackup_file="test/rackup/ci_select.ru"
+  rackup_file="test/rackup/sleep.ru"
 fi
 
 ip4=$PUMA_TEST_HOST4:$PUMA_TEST_PORT
@@ -171,5 +178,5 @@ StartPuma()
   fi
   printf "\nbundle exec bin/puma -q -b $bind $puma_args --control-url=tcp://$PUMA_CTRL --control-token=test $rackup_file\n\n"
   bundle exec bin/puma -q -b $bind $puma_args --control-url=tcp://$PUMA_CTRL --control-token=test $rackup_file &
-  sleep 6s
+  sleep 1s
 }

benchmarks/local/long_tail_hey.md

@@ -0,0 +1,100 @@
+# long_tail_hey
+
+`long_tail_hey` uses hey to create a stream of requests, and logs its report data, along with Puma data.
+
+The main purpose is to see Puma's behavior when it has more requests than available processing threads.
+
+The number of 'processing threads' is `<number of workers> * <max threads>`.
+
+Current code is set to run several `hey` commands, each increasing the number of connections.
+The configuration for each set is done in the `CONNECTION_MULT` array in `benchmarks/local/long_tail_hey.rb`.
+Change this array to suite your needs.
+
+A few examples:
+```text
+benchmarks/local/long_tail_hey.sh     -t5:5 -R100 -d0.2
+benchmarks/local/long_tail_hey.sh -w2 -t5:5 -R100 -d0.2
+benchmarks/local/long_tail_hey.sh -w4 -t5:5 -R100 -d0.2
+```
+
+## Arguments
+
+### Puma
+
+| Arg | Description | Example |
+| :-: | ----------- | ------- |
+| -t  | Threads     | 5:5     |
+| -w  | Workers     | 2       |
+| -r  | rackup file | \<path\>|
+| -C  | config file | \<path\>|
+
+### Hey
+
+| Arg | Description        | Example |
+| :-: | ------------------ | ------- |
+| -R  | req per conn       | 100     |
+| -k  | disable Keep-Alive | na      |
+| -d  | req delay          | float   |
+
+### Ruby
+
+| Arg | Description  |
+| :-: | ------------ |
+| -Y  | enable YJIT  |
+
+## ENV Settings
+
+| ENV       | Description (Default)                  |
+| --------  | -------------------------------------- |
+| HEY       | path to hey unless in PATH (hey)       |
+| HEY_CPUS  | number of cpu's for hey                |
+| PUMA_TEST_HOST4  | IPv4 host (127.0.0.1)           |
+| PUMA_TEST_HOST6  | IPv6 host (::1)                 |
+| PUMA_TEST_PORT   | port (40001)                    |
+
+
+## Example Output
+```text
+$ benchmarks/local/long_tail_hey.sh -w4 -t5:5 -R100 -d0.2
+
+bundle exec bin/puma -q -b tcp://127.0.0.1:40001 -S tmp/bench_test_puma.state -w4 -t5:5 --control-url=tcp://127.0.0.1:40010 --control-token=test test/rackup/sleep.ru
+
+[162886] Puma starting in cluster mode...
+[162886] * Puma version: 6.5.0 ("Sky's Version")
+[162886] * Ruby version: ruby 3.4.0dev (2024-11-26T17:58:43Z master c1dcd1d496) +PRISM [x86_64-linux]
+[162886] *  Min threads: 5
+[162886] *  Max threads: 5
+[162886] *  Environment: development
+[162886] *   Master PID: 162886
+[162886] *      Workers: 4
+[162886] *     Restarts: (✔) hot (✔) phased
+[162886] * Listening on http://127.0.0.1:40001
+[162886] Use Ctrl-C to stop
+[162886] * Starting control server on http://127.0.0.1:40010
+[162886] - Worker 0 (PID: 162895) booted in 0.01s, phase: 0
+[162886] - Worker 1 (PID: 162899) booted in 0.01s, phase: 0
+[162886] - Worker 2 (PID: 162902) booted in 0.01s, phase: 0
+[162886] - Worker 3 (PID: 162906) booted in 0.01s, phase: 0
+
+hey -c  20 -n  2000 -cpus 16  http://127.0.0.1:40001/sleep0.2
+hey -c  30 -n  3000 -cpus 16  http://127.0.0.1:40001/sleep0.2
+hey -c  40 -n  4000 -cpus 16  http://127.0.0.1:40001/sleep0.2
+hey -c  60 -n  6000 -cpus 16  http://127.0.0.1:40001/sleep0.2
+hey -c  80 -n  8000 -cpus 16  http://127.0.0.1:40001/sleep0.2
+
+
+Branch: 00-long-tail-hey Puma: -w4  -t5:5  dly 0.2                                         ─────── Worker Request Info ───────
+                         ─────────────────────────── Latency ───────────────────────────    Std        % deviation       Total
+Mult/Conn     req/sec      10%     25%     50%     75%     90%     95%     99%    100%      Dev        from 25.00%        Reqs
+1.00  20      97.0120    0.2010  0.2018  0.2042  0.2085  0.2098  0.2105  0.2127  0.2186    0.000    0.0  0.0  0.0  0.0    2000
+1.50  30      90.7293    0.2060  0.2130  0.2913  0.3983  0.4061  0.4098  0.4141  0.4204    0.935   -3.6 -1.6 -1.1  6.3    3000
+2.00  40      96.3451    0.4009  0.4023  0.4074  0.4132  0.4188  0.4507  0.5939  0.6144    0.597   -4.1  0.9  1.4  1.8    4000
+3.00  60      96.3643    0.6020  0.6051  0.6109  0.6178  0.6257  0.6396  0.7764  0.8148    0.261   -1.6  0.1  0.1  1.3    6000
+4.00  80      95.5128    0.6978  0.8031  0.8117  0.8291  0.9165  0.9970  1.0142  1.2035    0.555   -3.0 -1.2  1.5  2.7    8000
+
+[162886] - Gracefully shutting down workers...
+[162886] === puma shutdown: 2024-11-26 13:14:43 -0600 ===
+[162886] - Goodbye!
+
+ 4:27 Total Time
+```