Skip to content

Commit

Permalink
Use benchmarks.json file for benchmarking
Browse files Browse the repository at this point in the history 
Had to fix a few bugs in that file for invalid expressions.  I also
added a new one for function application.
  • Loading branch information
@jamesls
jamesls committed Mar 15, 2016
1 parent 849ba35 commit 072fb51
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 28 deletions.
48 changes: 26 additions & 22 deletions perf/perftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@
from jmespath.lexer import Lexer


DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cases')
BENCHMARK_FILE = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'tests',
'compliance',
'benchmarks.json')
APPROX_RUN_TIME = 0.5


Expand All @@ -30,16 +34,21 @@ def run_tests(tests):
given = test['given']
expression = test['expression']
result = test['result']
should_search = test['bench_type'] == 'full'
lex_time = _lex_time(expression)
parse_time = _parse_time(expression)
search_time = _search_time(expression, given)
combined_time = _combined_time(expression, given, result)
if should_search:
search_time = _search_time(expression, given)
combined_time = _combined_time(expression, given, result)
else:
search_time = 0
combined_time = 0
sys.stdout.write(
"lex_time: %.5fms, parse_time: %.5fms, search_time: %.5fms "
"combined_time: %.5fms " % (1000 * lex_time,
1000 * parse_time,
1000 * search_time,
1000 * combined_time))
"lex_time: %10.5fus, parse_time: %10.5fus, search_time: %10.5fus "
"combined_time: %10.5fus " % (1000000 * lex_time,
1000000 * parse_time,
1000000 * search_time,
1000000 * combined_time))
sys.stdout.write("name: %s\n" % test['name'])


Expand Down Expand Up @@ -129,28 +138,23 @@ def load_tests(filename):

def _add_cases(data, loaded, filename):
for case in data['cases']:
current = {'description': data.get('description', filename),
'given': data['given'],
'name': case.get('name', case['expression']),
'expression': case['expression'],
'result': case.get('result')}
current = {
'given': data['given'],
'name': case.get('comment', case['expression']),
'expression': case['expression'],
'result': case.get('result'),
'bench_type': case['bench'],
}
loaded.append(current)
return loaded


def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--directory', default=DIRECTORY)
parser.add_argument('-f', '--filename')
parser.add_argument('-f', '--filename', default=BENCHMARK_FILE)
args = parser.parse_args()
collected_tests = []
if args.filename:
collected_tests.extend(load_tests(args.filename))
else:
for filename in os.listdir(args.directory):
if filename.endswith('.json'):
full_path = os.path.join(args.directory, filename)
collected_tests.extend(load_tests(full_path))
collected_tests.extend(load_tests(args.filename))
run_tests(collected_tests)


Expand Down
36 changes: 30 additions & 6 deletions tests/compliance/benchmarks.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,27 +32,41 @@
}
}
}
},
"b": true,
"c": {
"d": true
}
},
"cases": [
{
"comment": "simple field",
"expression": "a",
"expression": "b",
"result": true,
"bench": "full"
},
{
"comment": "simple subexpression",
"expression": "a.b",
"expression": "c.d",
"result": true,
"bench": "full"
},
{
"comment": "deep field selection",
"comment": "deep field selection no match",
"expression": "a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s",
"result": null,
"bench": "full"
},
{
"comment": "deep field selection",
"expression": "a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p",
"result": true,
"bench": "full"
},
{
"comment": "simple or",
"expression": "not_there || a",
"expression": "not_there || b",
"result": true,
"bench": "full"
}
]
Expand All @@ -67,21 +81,31 @@
{
"comment": "deep ands",
"expression": "a && b && c && d && e && f && g && h && i && j && k && l && m && n && o && p && q && r && s && t && u && v && w && x && y && z",
"result": 25,
"bench": "full"
},
{
"comment": "deep ors",
"expression": "z || y || x || w || v || u || t || s || r || q || p || o || n || m || l || k || j || i || h || g || f || e || d || c || b || a",
"result": 25,
"bench": "full"
},
{
"comment": "lots of summing",
"expression": "sum(z, y, x, w, v, u, t, s, r, q, p, o, n, m, l, k, j, i, h, g, f, e, d, c, b, a)",
"expression": "sum([z, y, x, w, v, u, t, s, r, q, p, o, n, m, l, k, j, i, h, g, f, e, d, c, b, a])",
"result": 325,
"bench": "full"
},
{
"comment": "lots of function application",
"expression": "sum([z, sum([y, sum([x, sum([w, sum([v, sum([u, sum([t, sum([s, sum([r, sum([q, sum([p, sum([o, sum([n, sum([m, sum([l, sum([k, sum([j, sum([i, sum([h, sum([g, sum([f, sum([e, sum([d, sum([c, sum([b, a])])])])])])])])])])])])])])])])])])])])])])])])])",
"result": 325,
"bench": "full"
},
{
"comment": "lots of multi list",
"expression": "[z, y, x, w, v, u, t, s, r, q, p, o, n, m, l, k, j, i, h, g, f, e, d, c, b, a]",
"result": [25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0],
"bench": "full"
}
]
Expand Down Expand Up @@ -116,7 +140,7 @@
},
{
"comment": "filter projection",
"expression": "foo[bar > baz][qux > baz]",
"expression": "foo[?bar > baz][?qux > baz]",
"bench": "parse"
}
]
Expand Down

0 comments on commit 072fb51

Please sign in to comment.