Allow running the benchmarks the old way.

Author: ericsnowcurrently

benchmarks/.libs/legacyutils.py

@@ -0,0 +1,21 @@
+import sys
+
+
+def maybe_handle_legacy(bench_func, *args, loopsarg='loops', legacyarg=None):
+    if '--legacy' not in sys.argv:
+        return
+    argv = list(sys.argv[1:])
+    argv.remove('--legacy')
+
+    kwargs = {}
+    if legacyarg:
+        kwargs[legacyarg] = True
+    if argv:
+        assert loopsarg
+        kwargs[loopsarg] = int(argv[0])
+
+    _, times = bench_func(*args, **kwargs)
+    if len(argv) > 1:
+        json.dump(times, open(argv[1], 'w'))
+
+    sys.exit(0)

benchmarks/bm_aiohttp/legacyutils.py

@@ -0,0 +1 @@
+../.libs/legacyutils.py

benchmarks/bm_aiohttp/run_benchmark.py

@@ -13,7 +13,15 @@
 ARGV = [sys.executable, "serve.py"]
 
 
+#############################
+# benchmarks
+
 def bench_aiohttp_requests(loops=3000):
+    elapsed, _ = _bench_aiohttp_requests(loops)
+    return elapsed
+
+
+def _bench_aiohttp_requests(loops=3000, legacy=False):
     """Measure N HTTP requests to a local server.
 
     Note that the server is freshly started here.
@@ -27,17 +35,36 @@ def bench_aiohttp_requests(loops=3000):
     Hence this should be used with bench_time_func()
     insted of bench_func().
     """
+    start = pyperf.perf_counter()
     elapsed = 0
+    times = []
     with netutils.serving(ARGV, DATADIR, "127.0.0.1:8080"):
         requests_get = requests.get
-        for _ in range(loops):
+        for i in range(loops):
+            # This is a macro benchmark for a Python implementation
+            # so "elapsed" covers more than just how long a request takes.
             t0 = pyperf.perf_counter()
             requests_get("http://localhost:8080/blog/").text
-            elapsed += pyperf.perf_counter() - t0
-    return elapsed
+            t1 = pyperf.perf_counter()
+
+            elapsed += t1 - t0
+            times.append(t0)
+            if legacy and (i % 100 == 0):
+                print(i, t0 - start)
+        times.append(pyperf.perf_counter())
+        if legacy:
+            total = times[-1] - start
+            print("%.2fs (%.3freq/s)" % (total, loops / total))
+    return elapsed, times
 
 
+#############################
+# the script
+
 if __name__ == "__main__":
+    from legacyutils import maybe_handle_legacy
+    maybe_handle_legacy(_bench_aiohttp_requests, legacyarg='legacy')
+
     runner = pyperf.Runner()
     runner.metadata['description'] = "Test the performance of aiohttp"
     runner.bench_time_func("aiohttp", bench_aiohttp_requests)

benchmarks/bm_djangocms/legacyutils.py

@@ -0,0 +1 @@
+../.libs/legacyutils.py

benchmarks/bm_djangocms/run_benchmark.py

@@ -117,6 +117,11 @@ def _ensure_datadir(datadir, preserve=True):
 # benchmarks
 
 def bench_djangocms_requests(sitedir, loops=INNER_LOOPS):
+    elapsed, _ = _bench_djangocms_requests(loops)
+    return elapsed
+
+
+def _bench_djangocms_requests(sitedir, loops=INNER_LOOPS, legacy=False):
     """Measure N HTTP requests to a local server.
 
     Note that the server is freshly started here.
@@ -130,14 +135,26 @@ def bench_djangocms_requests(sitedir, loops=INNER_LOOPS):
     Hence this should be used with bench_time_func()
     insted of bench_func().
     """
+    start = pyperf.perf_counter()
     elapsed = 0
+    times = []
     with netutils.serving(ARGV_SERVE, sitedir, "127.0.0.1:8000"):
-        requests_get = requests.get
-        for _ in range(loops):
+        for i in range(loops):
+            # This is a macro benchmark for a Python implementation
+            # so "elapsed" covers more than just how long a request takes.
             t0 = pyperf.perf_counter()
-            requests_get("http://localhost:8000/").text
-            elapsed += pyperf.perf_counter() - t0
-    return elapsed
+            requests.get("http://localhost:8000/").text
+            t1 = pyperf.perf_counter()
+
+            elapsed += t1 - t0
+            times.append(t0)
+            if legacy and (i % 100 == 0):
+                print(i, t0 - start)
+        times.append(pyperf.perf_counter())
+        if legacy:
+            total = times[-1] - start
+            print("%.2fs (%.3freq/s)" % (total, loops / total))
+    return elapsed, times
 
 
 # We can't set "add_cmdline_args" on pyperf.Runner
@@ -157,48 +174,48 @@ def add_worker_args(cmd, _):
         )
 
 
+#############################
+# the script
+
 if __name__ == "__main__":
     """
     Usage:
-        python djangocms.py
-        python djangocms.py --setup DATADIR
-        python djangocms.py --serve DATADIR
+        python benchmarks/bm_djangocms/run_benchmark.py
+        python benchmarks/bm_djangocms/run_benchmark.py --setup DIR
+        python benchmarks/bm_djangocms/run_benchmark.py --serve DIR
 
     The first form creates a temporary directory, sets up djangocms in it,
     serves out of it, and removes the directory.
     The second form sets up a djangocms installation in the given directory.
-    The third form runs a benchmark out of an already-set-up directory
+    The third form runs the benchmark out of an already-set-up directory
     The second and third forms are useful if you want to benchmark the
     initial migration phase separately from the second serving phase.
     """
     runner = _Runner()
     runner.metadata['description'] = "Test the performance of a Django data migration"
 
     # Parse the CLI args.
-    runner.argparser.add_argument("--setup", action="store_const", const=True)
+    runner.argparser.add_argument("--legacy", action='store_true')
     group = runner.argparser.add_mutually_exclusive_group()
+    group.add_argument("--setup")
     group.add_argument("--serve")
-    group.add_argument("datadir", nargs="?")
     args = runner.argparser.parse_args()
 
-    if args.serve is not None:
+    if args.setup is not None:
+        args.datadir = args.setup
+        args.setup = True
+        args.serve = False
+    elif args.serve is not None:
         args.datadir = args.serve
+        args.setup = False
         args.serve = True
-        if not args.setup:
-            args.setup = False
-            if not args.datadir:
-                runner.argparser.error("missing datadir")
-            elif not os.path.exists(args.datadir):
-                cmd = f"{sys.executable} {sys.argv[0]} --setup {args.datadir}?"
-                sys.exit(f"ERROR: Did you forget to run {cmd}?")
-        default = False
-    elif args.setup is not None:
-        args.serve = False
-        default = False
+        if not os.path.exists(args.datadir):
+            cmd = f"{sys.executable} {sys.argv[0]} --setup {args.datadir}?"
+            sys.exit(f"ERROR: Did you forget to run {cmd}?")
     else:
+        args.datadir = None
         args.setup = True
         args.serve = True
-        default = True
 
     # DjangoCMS looks for Python on $PATH?
     _ensure_python_on_PATH()
@@ -209,8 +226,9 @@ def add_worker_args(cmd, _):
         # First, set up the site.
         if args.setup:
             sitedir, elapsed = setup(datadir)
-            print("%.2fs to initialize db" % (elapsed,))
-            print(f"site created in {sitedir}")
+            if args.legacy:
+                print("%.2fs to initialize db" % (elapsed,))
+                print(f"site created in {sitedir}")
             if not args.serve:
                 print(f"now run {sys.executable} {sys.argv[0]} --serve {datadir}")
         else:
@@ -219,6 +237,11 @@ def add_worker_args(cmd, _):
 
         # Then run the benchmark.
         if args.serve:
+            if args.legacy:
+                from legacyutils import maybe_handle_legacy
+                maybe_handle_legacy(_bench_djangocms_requests, sitedir, legacyarg='legacy')
+                sys.exit(0)
+
             runner.datadir = datadir
 
             def time_func(loops, *args):

benchmarks/bm_flaskblogging/legacyutils.py

@@ -0,0 +1 @@
+../.libs/legacyutils.py

benchmarks/bm_flaskblogging/run_benchmark.py

@@ -17,6 +17,11 @@
 # benchmarks
 
 def bench_flask_requests(loops=1800):
+    elapsed, _ = _bench_flask_requests(loops)
+    return elapsed
+
+
+def _bench_flask_requests(loops=1800, legacy=False):
     """Measure N HTTP requests to a local server.
 
     Note that the server is freshly started here.
@@ -30,17 +35,36 @@ def bench_flask_requests(loops=1800):
     Hence this should be used with bench_time_func()
     insted of bench_func().
     """
+    start = pyperf.perf_counter()
     elapsed = 0
+    times = []
     with netutils.serving(ARGV, DATADIR, "127.0.0.1:8000"):
         requests_get = requests.get
-        for _ in range(loops):
+        for i in range(loops):
+            # This is a macro benchmark for a Python implementation
+            # so "elapsed" covers more than just how long a request takes.
             t0 = pyperf.perf_counter()
             requests_get("http://localhost:8000/blog/").text
-            elapsed += pyperf.perf_counter() - t0
-    return elapsed
+            t1 = pyperf.perf_counter()
 
+            elapsed += t1 - t0
+            times.append(t0)
+            if legacy and (i % 100 == 0):
+                print(i, t0 - start)
+        times.append(pyperf.perf_counter())
+        if legacy:
+            total = times[-1] - start
+            print("%.2fs (%.3freq/s)" % (total, loops / total))
+    return elapsed, times
+
+
+#############################
+# the script
 
 if __name__ == "__main__":
+    from legacyutils import maybe_handle_legacy
+    maybe_handle_legacy(_bench_flask_requests, legacyarg='legacy')
+
     runner = pyperf.Runner()
     runner.metadata['description'] = "Test the performance of flask"
     runner.bench_time_func("flaskblogging", bench_flask_requests)

benchmarks/bm_gevent_hub/run_benchmark.py

@@ -83,7 +83,9 @@ def bench_switch(loops=1000):
         for _ in range(loops):
             t0 = pyperf.perf_counter()
             child_switch()
-            elapsed += pyperf.perf_counter() - t0
+            t1 = pyperf.perf_counter()
+
+            elapsed += t1 - t0
         return elapsed
 
 
@@ -97,7 +99,9 @@ def bench_wait_ready(loops=1000):
         for _ in range(loops):
             t0 = pyperf.perf_counter()
             hub_wait(watcher)
-            elapsed += pyperf.perf_counter() - t0
+            t1 = pyperf.perf_counter()
+
+            elapsed += t1 - t0
         return elapsed
 
 
@@ -145,9 +149,19 @@ def bench_wait_func_ready(loops=1000):
 }
 
 
+#############################
+# the script
+
 if __name__ == "__main__":
+    import sys
+    if '--legacy' in sys.argv:
+        for i in range(10000):
+            bench_switch()
+        sys.exit(0)
+
     runner = pyperf.Runner()
     runner.metadata['description'] = "Test the performance of gevent"
+    runner.argparser.add_argument("--legacy", action='store_true')
     runner.argparser.add_argument("benchmark", nargs="?",
                                   choices=sorted(BENCHMARKS),
                                   default="gevent_hub")

benchmarks/bm_gunicorn/legacyutils.py

@@ -0,0 +1 @@
+../.libs/legacyutils.py

benchmarks/bm_gunicorn/run_benchmark.py

@@ -23,7 +23,15 @@
 ]
 
 
+#############################
+# benchmarks
+
 def bench_gunicorn(loops=3000):
+    elapsed, _ = _bench_gunicorn(loops)
+    return elapsed
+
+
+def _bench_gunicorn(loops=3000, legacy=False):
     """Measure N HTTP requests to a local server.
 
     Note that the server is freshly started here.
@@ -37,17 +45,36 @@ def bench_gunicorn(loops=3000):
     Hence this should be used with bench_time_func()
     insted of bench_func().
     """
+    start = pyperf.perf_counter()
     elapsed = 0
+    times = []
     with netutils.serving(ARGV, DATADIR, ADDR):
         requests_get = requests.get
-        for _ in range(loops):
+        for i in range(loops):
+            # This is a macro benchmark for a Python implementation
+            # so "elapsed" covers more than just how long a request takes.
             t0 = pyperf.perf_counter()
             requests_get("http://localhost:8000/blog/").text
-            elapsed += pyperf.perf_counter() - t0
-    return elapsed
+            t1 = pyperf.perf_counter()
+
+            elapsed += t1 - t0
+            times.append(t0)
+            if legacy and (i % 100 == 0):
+                print(i, t0 - start)
+        times.append(pyperf.perf_counter())
+        if legacy:
+            total = times[-1] - start
+            print("%.2fs (%.3freq/s)" % (total, loops / total))
+    return elapsed, times
 
 
+#############################
+# the script
+
 if __name__ == "__main__":
+    from legacyutils import maybe_handle_legacy
+    maybe_handle_legacy(_bench_gunicorn, legacyarg='legacy')
+
     runner = pyperf.Runner()
     runner.metadata['description'] = "Test the performance of gunicorn"
     runner.bench_time_func("gunicorn", bench_gunicorn)

benchmarks/bm_json/legacyutils.py

@@ -0,0 +1 @@
+../.libs/legacyutils.py