Skip to content

Commit 31cb241

Browse files
filipecosta90filipecosta90
committed
Include keyspace range checks on flow tests
1 parent 82dba6a commit 31cb241

File tree

2 files changed

+210
-49
lines changed

2 files changed

+210
-49
lines changed

‎tests/include.py

+59-40
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,24 @@
77
TLS_CACERT = os.environ.get("TLS_CACERT", "")
88

99

10-
def assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count,
10+
def assert_keyspace_range(env, key_max, key_min, master_nodes_connections):
11+
expected_keyspace_range = key_max - key_min + 1
12+
overall_keyspace_range = agg_keyspace_range(master_nodes_connections)
13+
# assert we have the expected keyspace range
14+
env.assertEqual(expected_keyspace_range, overall_keyspace_range)
15+
16+
def assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
1117
overall_request_count):
12-
failed_asserts = env.getNumberOfFailedAssertion()
13-
try:
14-
# assert correct exit code
15-
env.assertTrue(memtier_ok == True)
16-
# assert we have all outputs
17-
env.assertTrue(os.path.isfile('{0}/mb.stdout'.format(config.results_dir)))
18-
env.assertTrue(os.path.isfile('{0}/mb.stderr'.format(config.results_dir)))
19-
env.assertTrue(os.path.isfile('{0}/mb.json'.format(config.results_dir)))
20-
# assert we have the expected request count
21-
env.assertEqual(overall_expected_request_count, overall_request_count)
22-
finally:
23-
if env.getNumberOfFailedAssertion() > failed_asserts:
24-
debugPrintMemtierOnError(config, env)
18+
# assert correct exit code
19+
env.assertTrue(memtier_ok == True)
20+
# assert we have all outputs
21+
env.assertTrue(os.path.isfile('{0}/mb.stdout'.format(config.results_dir)))
22+
env.assertTrue(os.path.isfile('{0}/mb.stderr'.format(config.results_dir)))
23+
env.assertTrue(os.path.isfile('{0}/mb.json'.format(config.results_dir)))
24+
25+
# assert we have the expected request count
26+
env.assertEqual(overall_expected_request_count, overall_request_count)
27+
2528

2629
def add_required_env_arguments(benchmark_specs, config, env, master_nodes_list):
2730
# check if environment is cluster
@@ -36,39 +39,55 @@ def add_required_env_arguments(benchmark_specs, config, env, master_nodes_list):
3639
config['redis_process_port'] = master_nodes_list[0]['port']
3740

3841

39-
def debugPrintMemtierOnError(config, env):
40-
with open('{0}/mb.stderr'.format(config.results_dir)) as stderr:
41-
env.debugPrint("### PRINTING STDERR OUTPUT OF MEMTIER ON FAILURE ###", True)
42-
env.debugPrint("### mb.stderr file location: {0}".format('{0}/mb.stderr'.format(config.results_dir)), True)
43-
for line in stderr:
44-
env.debugPrint(line.rstrip(), True)
45-
with open('{0}/mb.stdout'.format(config.results_dir)) as stderr:
46-
env.debugPrint("### PRINTING STDOUT OUTPUT OF MEMTIER ON FAILURE ###", True)
47-
env.debugPrint("### mb.stdout file location: {0}".format('{0}/mb.stdout'.format(config.results_dir)), True)
48-
for line in stderr:
49-
env.debugPrint(line.rstrip(), True)
50-
51-
if not env.isCluster():
52-
if env.envRunner is not None:
53-
log_file = os.path.join(env.envRunner.dbDirPath, env.envRunner._getFileName('master', '.log'))
54-
with open(log_file) as redislog:
55-
env.debugPrint("### REDIS LOG ###", True)
56-
env.debugPrint(
57-
"### log_file file location: {0}".format(log_file), True)
58-
for line in redislog:
59-
env.debugPrint(line.rstrip(), True)
60-
61-
62-
def get_expected_request_count(config):
42+
def debugPrintMemtierOnError(config, env, memtier_ok):
43+
if not memtier_ok:
44+
with open('{0}/mb.stderr'.format(config.results_dir)) as stderr:
45+
env.debugPrint("### PRINTING STDERR OUTPUT OF MEMTIER ON FAILURE ###", True)
46+
env.debugPrint("### mb.stderr file location: {0}".format('{0}/mb.stderr'.format(config.results_dir)), True)
47+
for line in stderr:
48+
env.debugPrint(line.rstrip(), True)
49+
50+
with open('{0}/mb.stdout'.format(config.results_dir)) as stderr:
51+
env.debugPrint("### PRINTING STDERR OUTPUT OF MEMTIER ON FAILURE ###", True)
52+
env.debugPrint("### mb.stderr file location: {0}".format('{0}/mb.stdout'.format(config.results_dir)), True)
53+
for line in stderr:
54+
env.debugPrint(line.rstrip(), True)
55+
56+
if not env.isCluster():
57+
if env.envRunner is not None:
58+
log_file = os.path.join(env.envRunner.dbDirPath, env.envRunner._getFileName('master', '.log'))
59+
with open(log_file) as redislog:
60+
env.debugPrint("### REDIS LOG ###", True)
61+
env.debugPrint(
62+
"### log_file file location: {0}".format(log_file), True)
63+
for line in redislog:
64+
env.debugPrint(line.rstrip(), True)
65+
66+
67+
def get_expected_request_count(config,key_minimum=0,key_maximum=1000000):
6368
result = -1
6469
if 'memtier_benchmark' in config:
6570
mt = config['memtier_benchmark']
6671
if 'threads' in mt and 'clients' in mt and 'requests' in mt:
67-
result = config['memtier_benchmark']['threads'] * config['memtier_benchmark']['clients'] * \
68-
config['memtier_benchmark']['requests']
72+
if mt['requests'] != 'allkeys':
73+
result = mt['threads'] * mt['clients'] * mt['requests']
74+
else:
75+
result = key_maximum - key_minimum + 1
6976
return result
7077

7178

79+
80+
def agg_keyspace_range(master_nodes_connections):
81+
overall_keyspace_range = 0
82+
for master_connection in master_nodes_connections:
83+
shard_reply = master_connection.execute_command("INFO", "KEYSPACE")
84+
shard_count = 0
85+
if 'db0' in shard_reply:
86+
if 'keys' in shard_reply['db0']:
87+
shard_count = int(shard_reply['db0']['keys'])
88+
overall_keyspace_range = overall_keyspace_range + shard_count
89+
return overall_keyspace_range
90+
7291
def agg_info_commandstats(master_nodes_connections, merged_command_stats):
7392
overall_request_count = 0
7493
for master_connection in master_nodes_connections:

‎tests/tests_oss_simple_flow.py

+151-9
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,135 @@
55
from mbdirector.runner import RunConfig
66

77

8+
def test_preload_and_set_get(env):
9+
key_max = 500000
10+
key_min = 1
11+
benchmark_specs = {"name": env.testName, "args": ['--pipeline=10','--ratio=1:0','--key-pattern=P:P','--key-minimum={}'.format(key_min),'--key-maximum={}'.format(key_max)]}
12+
addTLSArgs(benchmark_specs, env)
13+
config = get_default_memtier_config(threads=2, clients=10, requests='allkeys')
14+
master_nodes_list = env.getMasterNodesList()
15+
overall_expected_request_count = get_expected_request_count(config,key_min, key_max)
16+
17+
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
18+
19+
# Create a temporary directory
20+
test_dir = tempfile.mkdtemp()
21+
22+
config = RunConfig(test_dir, env.testName, config, {})
23+
ensure_clean_benchmark_folder(config.results_dir)
24+
25+
benchmark = Benchmark.from_json(config, benchmark_specs)
26+
27+
# benchmark.run() returns True if the return code of memtier_benchmark was 0
28+
memtier_ok = benchmark.run()
29+
debugPrintMemtierOnError(config, env, memtier_ok)
30+
31+
master_nodes_connections = env.getOSSMasterNodesConnectionList()
32+
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
33+
assert_keyspace_range(env, key_max, key_min, master_nodes_connections)
34+
35+
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
36+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
37+
overall_request_count)
38+
json_filename = '{0}/mb.json'.format(config.results_dir)
39+
40+
for master_connection in master_nodes_connections:
41+
master_connection.execute_command("CONFIG", "RESETSTAT")
42+
43+
benchmark_specs = {"name": env.testName, "args": ['--pipeline=10','--ratio=1:1','--key-pattern=R:R','--key-minimum={}'.format(key_min),'--key-maximum={}'.format(key_max)]}
44+
addTLSArgs(benchmark_specs, env)
45+
config = get_default_memtier_config(threads=2, clients=10, requests=200000)
46+
master_nodes_list = env.getMasterNodesList()
47+
overall_expected_request_count = get_expected_request_count(config,key_min, key_max)
48+
49+
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
50+
51+
# Create a temporary directory
52+
test_dir = tempfile.mkdtemp()
53+
54+
config = RunConfig(test_dir, env.testName, config, {})
55+
ensure_clean_benchmark_folder(config.results_dir)
56+
57+
benchmark = Benchmark.from_json(config, benchmark_specs)
58+
59+
# benchmark.run() returns True if the return code of memtier_benchmark was 0
60+
memtier_ok = benchmark.run()
61+
debugPrintMemtierOnError(config, env, memtier_ok)
62+
63+
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
64+
assert_keyspace_range(env, key_max, key_min, master_nodes_connections)
65+
66+
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
67+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
68+
overall_request_count)
69+
json_filename = '{0}/mb.json'.format(config.results_dir)
70+
71+
72+
def test_default_set(env):
73+
key_max = 500000
74+
key_min = 1
75+
benchmark_specs = {"name": env.testName, "args": ['--pipeline=10','--ratio=1:0','--key-pattern=P:P','--key-minimum={}'.format(key_min),'--key-maximum={}'.format(key_max)]}
76+
addTLSArgs(benchmark_specs, env)
77+
config = get_default_memtier_config(threads=2, clients=10, requests='allkeys')
78+
master_nodes_list = env.getMasterNodesList()
79+
overall_expected_request_count = get_expected_request_count(config,key_min, key_max)
80+
81+
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
82+
83+
# Create a temporary directory
84+
test_dir = tempfile.mkdtemp()
85+
86+
config = RunConfig(test_dir, env.testName, config, {})
87+
ensure_clean_benchmark_folder(config.results_dir)
88+
89+
benchmark = Benchmark.from_json(config, benchmark_specs)
90+
91+
# benchmark.run() returns True if the return code of memtier_benchmark was 0
92+
memtier_ok = benchmark.run()
93+
debugPrintMemtierOnError(config, env, memtier_ok)
94+
master_nodes_connections = env.getOSSMasterNodesConnectionList()
95+
96+
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
97+
assert_keyspace_range(env, key_max, key_min, master_nodes_connections)
98+
99+
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
100+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
101+
overall_request_count)
102+
json_filename = '{0}/mb.json'.format(config.results_dir)
103+
104+
# ensure if we run again on a different key pattern the dataset doesn't grow
105+
for master_connection in master_nodes_connections:
106+
master_connection.execute_command("CONFIG", "RESETSTAT")
107+
108+
benchmark_specs = {"name": env.testName, "args": ['--pipeline=10','--ratio=1:0','--key-pattern=R:R','--key-minimum={}'.format(key_min),'--key-maximum={}'.format(key_max)]}
109+
addTLSArgs(benchmark_specs, env)
110+
config = get_default_memtier_config(threads=2, clients=10, requests=200000)
111+
master_nodes_list = env.getMasterNodesList()
112+
overall_expected_request_count = get_expected_request_count(config,key_min, key_max)
113+
114+
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
115+
116+
# Create a temporary directory
117+
test_dir = tempfile.mkdtemp()
118+
119+
config = RunConfig(test_dir, env.testName, config, {})
120+
ensure_clean_benchmark_folder(config.results_dir)
121+
122+
benchmark = Benchmark.from_json(config, benchmark_specs)
123+
124+
# benchmark.run() returns True if the return code of memtier_benchmark was 0
125+
memtier_ok = benchmark.run()
126+
debugPrintMemtierOnError(config, env, memtier_ok)
127+
128+
master_nodes_connections = env.getOSSMasterNodesConnectionList()
129+
merged_command_stats = {'cmdstat_set': {'calls': 0}}
130+
assert_keyspace_range(env, key_max, key_min, master_nodes_connections)
131+
132+
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
133+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
134+
overall_request_count)
135+
json_filename = '{0}/mb.json'.format(config.results_dir)
136+
8137
def test_default_set_get(env):
9138
benchmark_specs = {"name": env.testName, "args": []}
10139
addTLSArgs(benchmark_specs, env)
@@ -24,11 +153,13 @@ def test_default_set_get(env):
24153

25154
# benchmark.run() returns True if the return code of memtier_benchmark was 0
26155
memtier_ok = benchmark.run()
156+
debugPrintMemtierOnError(config, env, memtier_ok)
27157

28158
master_nodes_connections = env.getOSSMasterNodesConnectionList()
29159
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
30160
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
31-
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
161+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
162+
overall_request_count)
32163

33164

34165
def test_default_set_get_with_print_percentiles(env):
@@ -52,11 +183,13 @@ def test_default_set_get_with_print_percentiles(env):
52183

53184
# benchmark.run() returns True if the return code of memtier_benchmark was 0
54185
memtier_ok = benchmark.run()
186+
debugPrintMemtierOnError(config, env, memtier_ok)
55187

56188
master_nodes_connections = env.getOSSMasterNodesConnectionList()
57189
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
58190
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
59-
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
191+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
192+
overall_request_count)
60193
json_filename = '{0}/mb.json'.format(config.results_dir)
61194

62195
hdr_files_sufix = ["_FULL_RUN_1","_SET_command_run_1","_GET_command_run_1"]
@@ -99,11 +232,13 @@ def test_default_set_get_1_1(env):
99232

100233
# benchmark.run() returns True if the return code of memtier_benchmark was 0
101234
memtier_ok = benchmark.run()
235+
debugPrintMemtierOnError(config, env, memtier_ok)
102236

103237
master_nodes_connections = env.getOSSMasterNodesConnectionList()
104238
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
105239
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
106-
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
240+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
241+
overall_request_count)
107242

108243
# assert same number of gets and sets
109244
env.assertEqual(merged_command_stats['cmdstat_set']['calls'], merged_command_stats['cmdstat_get']['calls'])
@@ -130,11 +265,13 @@ def test_default_set_get_3_runs(env):
130265

131266
# benchmark.run() returns True if the return code of memtier_benchmark was 0
132267
memtier_ok = benchmark.run()
268+
debugPrintMemtierOnError(config, env, memtier_ok)
133269

134270
master_nodes_connections = env.getOSSMasterNodesConnectionList()
135271
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
136272
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
137-
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
273+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
274+
overall_request_count)
138275

139276

140277
def test_default_arbitrary_command_pubsub(env):
@@ -143,6 +280,7 @@ def test_default_arbitrary_command_pubsub(env):
143280
addTLSArgs(benchmark_specs, env)
144281
config = get_default_memtier_config()
145282
master_nodes_list = env.getMasterNodesList()
283+
overall_expected_request_count = 0
146284

147285
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
148286

@@ -154,8 +292,9 @@ def test_default_arbitrary_command_pubsub(env):
154292

155293
benchmark = Benchmark.from_json(config, benchmark_specs)
156294

157-
if not benchmark.run():
158-
debugPrintMemtierOnError(config, env)
295+
# benchmark.run() returns True if the return code of memtier_benchmark was 0
296+
memtier_ok = benchmark.run()
297+
debugPrintMemtierOnError(config, env, memtier_ok)
159298

160299

161300
def test_default_arbitrary_command_set(env):
@@ -178,11 +317,13 @@ def test_default_arbitrary_command_set(env):
178317

179318
# benchmark.run() returns True if the return code of memtier_benchmark was 0
180319
memtier_ok = benchmark.run()
320+
debugPrintMemtierOnError(config, env, memtier_ok)
181321

182322
master_nodes_connections = env.getOSSMasterNodesConnectionList()
183323
merged_command_stats = {'cmdstat_set': {'calls': 0}}
184324
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
185-
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
325+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
326+
overall_request_count)
186327

187328

188329
def test_default_arbitrary_command_hset(env):
@@ -205,9 +346,10 @@ def test_default_arbitrary_command_hset(env):
205346

206347
# benchmark.run() returns True if the return code of memtier_benchmark was 0
207348
memtier_ok = benchmark.run()
349+
debugPrintMemtierOnError(config, env, memtier_ok)
208350

209351
master_nodes_connections = env.getOSSMasterNodesConnectionList()
210352
merged_command_stats = {'cmdstat_hset': {'calls': 0}}
211353
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
212-
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
213-
354+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
355+
overall_request_count)

0 commit comments

Comments
 (0)