A few fixes

README.rst

@@ -12,13 +12,20 @@ Installation
 
 Install the following:
 
-- Docker
-- Python 3
-- Numpy
+- `Docker<https://docs.docker.com/engine/installation/>` (tested with *17.09.0-ce*)
+- Python 3.5+ (tested with *3.5.2*)
+- `numpy<https://pypi.python.org/pypi/numpy>` (tested with *1.13.3*)
+- `wrk<https://github.com/wg/wrk>` (tested with *4.0.0*)
 
 Build the docker image containing the servers being tested by running
 ``./build.sh``.
 
 The benchmarks can then be ran with ``./run_benchmarks``.  Use
 ``./run_benchmarks --help`` for various options, including selective
 benchmark running.
+
+To run the http benchmarks and save results to ``./results.html``:
+
+.. code::
+
+  ./run_benchmarks --duration=60 --save-html=results.html http

build.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker build -t magic/benchmark $(dirname $0)
+docker build --no-cache -t magic/benchmark "$(dirname $0)"

run_benchmarks

@@ -293,11 +293,15 @@ def server_container_exists():
 def kill_server():
     if server_is_running():
         print('Shutting down server...')
-        subprocess.check_output(['docker', 'stop', 'magicbench'])
-
-    if server_container_exists():
-        print('Removing server container...')
-        subprocess.check_output(['docker', 'rm', 'magicbench'])
+        subprocess.check_output(['docker', 'stop', '-t10', 'magicbench'])
+    i = 0
+    while server_container_exists():
+        if i == 100:
+            subprocess.check_output(['docker', 'rm', '-f', 'magicbench'])
+        elif i == 110:
+            raise IOError("container still exists a minute after being sigkilled")
+        time.sleep(0.1)
+        i += 1
 
 
 def format_report(data, target_file):
@@ -438,9 +442,11 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--duration', '-D', default=30, type=int,
                         help='duration of each benchmark in seconds')
-    parser.add_argument('--benchmarks', type=str,
-                        help='comma-separated list of benchmarks to run ' +
-                             '(regular expressions are supported)')
+    parser.add_argument('type', type=str,
+                        choices={b['name'].split('-', 1)[0] for b in benchmarks},
+                        help='type of benchmark to run')
+    parser.add_argument('benchmark_patterns', type=str, nargs='*',
+                        help='run benchmarks that match any of these regexes')
     parser.add_argument('--concurrency-levels', type=int, default=[10],
                         nargs='+',
                         help='a list of concurrency levels to use')
@@ -457,11 +463,15 @@ def main():
     if not os.path.exists(_socket):
         os.mkdir(_socket)
 
-    if args.benchmarks:
-        benchmarks_to_run = [re.compile(b) for b in args.benchmarks.split(',')]
+    if args.benchmark_patterns:
+        patterns = [re.compile(p) for p in args.benchmark_patterns]
     else:
-        benchmarks_to_run = [re.compile(re.escape(b['name']))
-                             for b in benchmarks]
+        patterns = [re.compile('')]
+    benchmarks_to_run = []
+    for benchmark in benchmarks:
+        type_, name = benchmark['name'].split('-', 1)
+        if type_ == args.type and any(p.match(name) for p in patterns):
+            benchmarks_to_run.append(benchmark)
 
     benchmarks_data = []
 
@@ -470,7 +480,7 @@ def main():
     for concurrency in sorted(args.concurrency_levels):
         for msgsize in sorted(args.payload_size_levels):
             variations.append({
-                'title': '{}kb messages, concurrency {}'.format(
+                'title': '{}KiB messages, concurrency {}'.format(
                     round(msgsize / 1024, 1), concurrency
                 ),
                 'concurrency': concurrency,
@@ -485,10 +495,7 @@ def main():
     warmup = ['--msize=1024', '--duration=10',
               '--concurrency={}'.format(warmup_concurrency)]
 
-    for benchmark in benchmarks:
-        if not any(b.match(benchmark['name']) for b in benchmarks_to_run):
-            continue
-
+    for benchmark in benchmarks_to_run:
         print(benchmark['title'])
         print('=' * len(benchmark['title']))
         print()

servers/requirements.txt

@@ -4,4 +4,4 @@ gevent==1.1.1
 tornado==4.3
 Twisted==16.1.1
 httptools==0.0.9
-uvloop==0.4.28
+uvloop==0.8.1