-
Notifications
You must be signed in to change notification settings - Fork 3
/
benchmark.py
125 lines (99 loc) · 4.14 KB
/
benchmark.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#!/usr/bin/env python
#
# Copyright 2014 Brian Quinlan
# See "LICENSE" file for details.
# pylint: disable=print-statement
"""Run benchmarks among the various execution modes."""
from __future__ import absolute_import
from __future__ import division
import argparse
import itertools
import tempfile
import test_runner
import timeit
HEADER = (
#pylint: disable=bad-continuation
" Interpreter Compiler JIT")
TRIALS_HEADER_FORMAT = (
#pylint: disable=bad-continuation
"============================================================================\n"
"Max Loop Nesting: %d\n"
"============================================================================")
TRIAL_FORMAT = (
#pylint: disable=bad-continuation
"Trial %2d: %6.2f %6.2f %6.2f")
TRIAL_SUMMARY_FORMAT = (
#pylint: disable=bad-continuation
"----------------------------------------------------------------------------\n"
"Total (range): %6.2f %6.2f (%0.2f-%0.2f) %6.2f (%0.2f-%0.2f)\n"
)
def time(mode, path, repeat, number):
times = timeit.repeat(
("returncode, _, stderr = run_brainfuck(['--mode=%s', %r]); " +
"assert returncode == 0, 'returncode = %%d, stderr = %%r' %% (" +
" returncode, stderr)") % (mode, path),
setup="from test_runner import run_brainfuck",
repeat=repeat,
number=number)
return min(times)
def time_all(trial_number, path, repeat, number):
"""Time all execution modes with the given file."""
interpreter_min_time = time('i', path, repeat, number)
compiler_min_time = time('cag', path, repeat, number)
jit_min_time = time('jit', path, repeat, number)
print TRIAL_FORMAT % (
trial_number,
interpreter_min_time,
compiler_min_time,
jit_min_time)
return interpreter_min_time, compiler_min_time, jit_min_time
def main(): # pylint: disable=missing-docstring
parser = argparse.ArgumentParser(
description='Benchmark the bf executable')
parser.add_argument(
'--repeat', '-r',
dest='repeat', action='store', type=int, default=5,
help='the number of times to collect before the minimum is taken')
parser.add_argument(
'--number', '-n',
dest='number', action='store', type=int, default=20,
help=('the number of times to run each command before collecting the '
'time'))
parser.add_argument(
'--trials', '-t',
dest='num_trials', action='store', type=int, default=20,
help=('the of randomly generated code samples to use for each loop '
'level'))
options = parser.parse_args()
print
print HEADER
for max_nested_loops in itertools.count():
print TRIALS_HEADER_FORMAT % max_nested_loops
times = []
for trial_number in range(options.num_trials):
brainfuck_code = test_runner.generate_brainfuck_code(
'<>+-[]\n', 1024 * 1024, max_nested_loops)
# pylint doesn't seem to understand with statements
# pylint: disable=bad-continuation
with tempfile.NamedTemporaryFile(
suffix='.b', delete=False) as brainfuck_source_file:
brainfuck_source_file.write(brainfuck_code)
brainfuck_source_file.close()
times.append(
time_all(trial_number+1,
brainfuck_source_file.name,
options.repeat,
options.number))
interpreter_mean = sum(i for (i, _, _) in times)
compiler_mean = sum(c for (_, c, _) in times)
jit_mean = sum(j for (_, _, j) in times)
jit_best_relative = min(j/i for (i, _, j) in times)
jit_worst_relative = max(j/i for (i, _, j) in times)
compiler_best_relative = min(c/i for (i, c, _) in times)
compiler_worst_relative = max(c/i for (i, c, _) in times)
print TRIAL_SUMMARY_FORMAT % (
interpreter_mean,
compiler_mean, compiler_best_relative, compiler_worst_relative,
jit_mean, jit_best_relative, jit_worst_relative)
if __name__ == '__main__':
main()