Skip to content

Commit 82dba6a

Browse files
ushacharUri Shachar
ushachar
and
Uri Shachar
authored
Always print out stderr/out output on test failures. (#207)
Also do some minor cleanups around unused args. Co-authored-by: Uri Shachar <[email protected]>
1 parent 78b0a9a commit 82dba6a

File tree

2 files changed

+44
-56
lines changed

2 files changed

+44
-56
lines changed

‎tests/include.py

+35-34
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,21 @@
77
TLS_CACERT = os.environ.get("TLS_CACERT", "")
88

99

10-
def assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
10+
def assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count,
1111
overall_request_count):
12-
# assert correct exit code
13-
env.assertTrue(memtier_ok == True)
14-
# assert we have all outputs
15-
env.assertTrue(os.path.isfile('{0}/mb.stdout'.format(config.results_dir)))
16-
env.assertTrue(os.path.isfile('{0}/mb.stderr'.format(config.results_dir)))
17-
env.assertTrue(os.path.isfile('{0}/mb.json'.format(config.results_dir)))
18-
19-
# assert we have the expected request count
20-
env.assertEqual(overall_expected_request_count, overall_request_count)
21-
12+
failed_asserts = env.getNumberOfFailedAssertion()
13+
try:
14+
# assert correct exit code
15+
env.assertTrue(memtier_ok == True)
16+
# assert we have all outputs
17+
env.assertTrue(os.path.isfile('{0}/mb.stdout'.format(config.results_dir)))
18+
env.assertTrue(os.path.isfile('{0}/mb.stderr'.format(config.results_dir)))
19+
env.assertTrue(os.path.isfile('{0}/mb.json'.format(config.results_dir)))
20+
# assert we have the expected request count
21+
env.assertEqual(overall_expected_request_count, overall_request_count)
22+
finally:
23+
if env.getNumberOfFailedAssertion() > failed_asserts:
24+
debugPrintMemtierOnError(config, env)
2225

2326
def add_required_env_arguments(benchmark_specs, config, env, master_nodes_list):
2427
# check if environment is cluster
@@ -33,29 +36,27 @@ def add_required_env_arguments(benchmark_specs, config, env, master_nodes_list):
3336
config['redis_process_port'] = master_nodes_list[0]['port']
3437

3538

36-
def debugPrintMemtierOnError(config, env, memtier_ok):
37-
if not memtier_ok:
38-
with open('{0}/mb.stderr'.format(config.results_dir)) as stderr:
39-
env.debugPrint("### PRINTING STDERR OUTPUT OF MEMTIER ON FAILURE ###", True)
40-
env.debugPrint("### mb.stderr file location: {0}".format('{0}/mb.stderr'.format(config.results_dir)), True)
41-
for line in stderr:
42-
env.debugPrint(line.rstrip(), True)
43-
44-
with open('{0}/mb.stdout'.format(config.results_dir)) as stderr:
45-
env.debugPrint("### PRINTING STDERR OUTPUT OF MEMTIER ON FAILURE ###", True)
46-
env.debugPrint("### mb.stderr file location: {0}".format('{0}/mb.stdout'.format(config.results_dir)), True)
47-
for line in stderr:
48-
env.debugPrint(line.rstrip(), True)
49-
50-
if not env.isCluster():
51-
if env.envRunner is not None:
52-
log_file = os.path.join(env.envRunner.dbDirPath, env.envRunner._getFileName('master', '.log'))
53-
with open(log_file) as redislog:
54-
env.debugPrint("### REDIS LOG ###", True)
55-
env.debugPrint(
56-
"### log_file file location: {0}".format(log_file), True)
57-
for line in redislog:
58-
env.debugPrint(line.rstrip(), True)
39+
def debugPrintMemtierOnError(config, env):
40+
with open('{0}/mb.stderr'.format(config.results_dir)) as stderr:
41+
env.debugPrint("### PRINTING STDERR OUTPUT OF MEMTIER ON FAILURE ###", True)
42+
env.debugPrint("### mb.stderr file location: {0}".format('{0}/mb.stderr'.format(config.results_dir)), True)
43+
for line in stderr:
44+
env.debugPrint(line.rstrip(), True)
45+
with open('{0}/mb.stdout'.format(config.results_dir)) as stderr:
46+
env.debugPrint("### PRINTING STDOUT OUTPUT OF MEMTIER ON FAILURE ###", True)
47+
env.debugPrint("### mb.stdout file location: {0}".format('{0}/mb.stdout'.format(config.results_dir)), True)
48+
for line in stderr:
49+
env.debugPrint(line.rstrip(), True)
50+
51+
if not env.isCluster():
52+
if env.envRunner is not None:
53+
log_file = os.path.join(env.envRunner.dbDirPath, env.envRunner._getFileName('master', '.log'))
54+
with open(log_file) as redislog:
55+
env.debugPrint("### REDIS LOG ###", True)
56+
env.debugPrint(
57+
"### log_file file location: {0}".format(log_file), True)
58+
for line in redislog:
59+
env.debugPrint(line.rstrip(), True)
5960

6061

6162
def get_expected_request_count(config):

‎tests/tests_oss_simple_flow.py

+9-22
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,11 @@ def test_default_set_get(env):
2424

2525
# benchmark.run() returns True if the return code of memtier_benchmark was 0
2626
memtier_ok = benchmark.run()
27-
debugPrintMemtierOnError(config, env, memtier_ok)
2827

2928
master_nodes_connections = env.getOSSMasterNodesConnectionList()
3029
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
3130
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
32-
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
33-
overall_request_count)
31+
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
3432

3533

3634
def test_default_set_get_with_print_percentiles(env):
@@ -54,13 +52,11 @@ def test_default_set_get_with_print_percentiles(env):
5452

5553
# benchmark.run() returns True if the return code of memtier_benchmark was 0
5654
memtier_ok = benchmark.run()
57-
debugPrintMemtierOnError(config, env, memtier_ok)
5855

5956
master_nodes_connections = env.getOSSMasterNodesConnectionList()
6057
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
6158
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
62-
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
63-
overall_request_count)
59+
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
6460
json_filename = '{0}/mb.json'.format(config.results_dir)
6561

6662
hdr_files_sufix = ["_FULL_RUN_1","_SET_command_run_1","_GET_command_run_1"]
@@ -103,13 +99,11 @@ def test_default_set_get_1_1(env):
10399

104100
# benchmark.run() returns True if the return code of memtier_benchmark was 0
105101
memtier_ok = benchmark.run()
106-
debugPrintMemtierOnError(config, env, memtier_ok)
107102

108103
master_nodes_connections = env.getOSSMasterNodesConnectionList()
109104
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
110105
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
111-
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
112-
overall_request_count)
106+
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
113107

114108
# assert same number of gets and sets
115109
env.assertEqual(merged_command_stats['cmdstat_set']['calls'], merged_command_stats['cmdstat_get']['calls'])
@@ -136,13 +130,11 @@ def test_default_set_get_3_runs(env):
136130

137131
# benchmark.run() returns True if the return code of memtier_benchmark was 0
138132
memtier_ok = benchmark.run()
139-
debugPrintMemtierOnError(config, env, memtier_ok)
140133

141134
master_nodes_connections = env.getOSSMasterNodesConnectionList()
142135
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
143136
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
144-
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
145-
overall_request_count)
137+
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
146138

147139

148140
def test_default_arbitrary_command_pubsub(env):
@@ -151,7 +143,6 @@ def test_default_arbitrary_command_pubsub(env):
151143
addTLSArgs(benchmark_specs, env)
152144
config = get_default_memtier_config()
153145
master_nodes_list = env.getMasterNodesList()
154-
overall_expected_request_count = 0
155146

156147
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
157148

@@ -163,9 +154,8 @@ def test_default_arbitrary_command_pubsub(env):
163154

164155
benchmark = Benchmark.from_json(config, benchmark_specs)
165156

166-
# benchmark.run() returns True if the return code of memtier_benchmark was 0
167-
memtier_ok = benchmark.run()
168-
debugPrintMemtierOnError(config, env, memtier_ok)
157+
if not benchmark.run():
158+
debugPrintMemtierOnError(config, env)
169159

170160

171161
def test_default_arbitrary_command_set(env):
@@ -188,13 +178,11 @@ def test_default_arbitrary_command_set(env):
188178

189179
# benchmark.run() returns True if the return code of memtier_benchmark was 0
190180
memtier_ok = benchmark.run()
191-
debugPrintMemtierOnError(config, env, memtier_ok)
192181

193182
master_nodes_connections = env.getOSSMasterNodesConnectionList()
194183
merged_command_stats = {'cmdstat_set': {'calls': 0}}
195184
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
196-
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
197-
overall_request_count)
185+
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
198186

199187

200188
def test_default_arbitrary_command_hset(env):
@@ -217,10 +205,9 @@ def test_default_arbitrary_command_hset(env):
217205

218206
# benchmark.run() returns True if the return code of memtier_benchmark was 0
219207
memtier_ok = benchmark.run()
220-
debugPrintMemtierOnError(config, env, memtier_ok)
221208

222209
master_nodes_connections = env.getOSSMasterNodesConnectionList()
223210
merged_command_stats = {'cmdstat_hset': {'calls': 0}}
224211
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
225-
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
226-
overall_request_count)
212+
assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
213+

0 commit comments

Comments
 (0)