forked from apache/cassandra-dtest
-
Notifications
You must be signed in to change notification settings - Fork 3
/
dtest_config.py
147 lines (125 loc) · 7.32 KB
/
dtest_config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
import subprocess
import os
import ccmlib.repository
import logging
from ccmlib.common import is_win, get_version_from_build
from pytest import UsageError
logger = logging.getLogger(__name__)
class DTestConfig:
def __init__(self):
self.sstable_format = "bti"
self.use_vnodes = True
self.configuration_yaml = None
self.use_off_heap_memtables = False
self.num_tokens = -1
self.data_dir_count = -1
self.force_execution_of_resource_intensive_tests = False
self.skip_resource_intensive_tests = False
self.only_resource_intensive_tests = False
self.cassandra_dir = None
self.cassandra_version = None
self.cassandra_version_from_build = None
self.delete_logs = False
self.execute_upgrade_tests = False
self.execute_upgrade_tests_only = False
self.disable_active_log_watching = False
self.keep_test_dir = False
self.keep_failed_test_dir = False
self.enable_jacoco_code_coverage = False
self.jemalloc_path = find_libjemalloc()
self.metatests = False
self.latest_config = False
def setup(self, config):
"""
Reads and validates configuration. Throws UsageError if configuration is invalid.
"""
self.metatests = config.getoption("--metatests")
if self.metatests:
self.cassandra_dir = os.path.join(os.getcwd(), "meta_tests/cassandra-dir-4.0-beta")
self.cassandra_version_from_build = self.get_version_from_build()
return
self.sstable_format = config.getoption("--sstable-format")
if self.sstable_format:
assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format)
self.use_vnodes = config.getoption("--use-vnodes")
self.configuration_yaml = config.getoption("--configuration-yaml")
self.use_off_heap_memtables = config.getoption("--use-off-heap-memtables")
self.num_tokens = config.getoption("--num-tokens")
self.data_dir_count = config.getoption("--data-dir-count-per-instance")
self.force_execution_of_resource_intensive_tests = config.getoption("--force-resource-intensive-tests")
self.skip_resource_intensive_tests = config.getoption("--skip-resource-intensive-tests")
self.only_resource_intensive_tests = config.getoption("--only-resource-intensive-tests")
cassandra_dir = config.getoption("--cassandra-dir") or config.getini("cassandra_dir")
if cassandra_dir is not None and cassandra_dir.strip() == "":
cassandra_dir = None
if cassandra_dir is not None:
self.cassandra_dir = os.path.expanduser(cassandra_dir)
self.cassandra_version = config.getoption("--cassandra-version")
if self.cassandra_version is not None and self.cassandra_dir is not None:
raise UsageError("Please remove --cassandra-version because Cassandra build directory is already "
"defined (by --cassandra-dir or in ini file)")
try:
self.cassandra_version_from_build = self.get_version_from_build()
except FileNotFoundError as fnfe:
raise UsageError("The Cassandra directory %s does not seem to be valid: %s" % (self.cassandra_dir, fnfe))
self.delete_logs = config.getoption("--delete-logs")
self.execute_upgrade_tests = config.getoption("--execute-upgrade-tests")
self.execute_upgrade_tests_only = config.getoption("--execute-upgrade-tests-only")
self.disable_active_log_watching = config.getoption("--disable-active-log-watching")
self.keep_test_dir = config.getoption("--keep-test-dir")
self.keep_failed_test_dir = config.getoption("--keep-failed-test-dir")
self.enable_jacoco_code_coverage = config.getoption("--enable-jacoco-code-coverage")
if self.cassandra_version is None and self.cassandra_version_from_build is None:
raise UsageError("Required dtest arguments were missing! You must provide either --cassandra-dir "
"or --cassandra-version. You can also set 'cassandra_dir' in pytest.ini. "
"Refer to the documentation or invoke the help with --help.")
version = self.cassandra_version or self.cassandra_version_from_build
if self.skip_resource_intensive_tests and \
(self.only_resource_intensive_tests or self.force_execution_of_resource_intensive_tests):
raise UsageError("--skip-resource-intensive-tests does not make any sense with either "
"--only-resource-intensive-tests or --force-resource-intensive-tests.")
# Check that use_off_heap_memtables is supported in this c* version
if self.use_off_heap_memtables and ("3.0" <= version < "3.4"):
raise UsageError("The selected Cassandra version %s doesn't support the provided option "
"--use-off-heap-memtables, see https://issues.apache.org/jira/browse/CASSANDRA-9472 "
"for details" % version)
self.latest_config = False if self.configuration_yaml is None else self.configuration_yaml.endswith("cassandra_latest.yaml")
self.apply_to_env(os.environ, "JVM_EXTRA_OPTS")
def apply_to_env(self, env, key="JVM_OPTS"):
current = env.get(key) or ""
if self.sstable_format:
default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format
if not current.__contains__("-Dcassandra.sstable.format.default"):
env.update({key: (env.get(key) or "") + default_sstable_format_prop})
else:
logger.debug("Skipped adding {} because it is already in the env key {}: {}".format(default_sstable_format_prop, key, current))
def get_version_from_build(self):
# There are times when we want to know the C* version we're testing against
# before we do any cluster. In the general case, we can't know that -- the
# test method could use any version it wants for self.cluster. However, we can
# get the version from build.xml in the C* repository specified by
# CASSANDRA_VERSION or CASSANDRA_DIR.
if self.cassandra_version is not None:
ccm_repo_cache_dir, _ = ccmlib.repository.setup(self.cassandra_version)
return get_version_from_build(ccm_repo_cache_dir)
elif self.cassandra_dir is not None:
return get_version_from_build(self.cassandra_dir)
# Determine the location of the libjemalloc jar so that we can specify it
# through environment variables when start Cassandra. This reduces startup
# time, making the dtests run faster.
def find_libjemalloc():
if is_win():
# let the normal bat script handle finding libjemalloc
return ""
this_dir = os.path.dirname(os.path.realpath(__file__))
script = os.path.join(this_dir, "findlibjemalloc.sh")
try:
p = subprocess.Popen([script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if stderr or not stdout:
return "-" # tells C* not to look for libjemalloc
else:
return stdout
except Exception as exc:
print("Failed to run script to prelocate libjemalloc ({}): {}".format(script, exc))
return ""