STAR-617-2: Changes from DSEDB - DO NOT MERGE

bootstrap_test.py

@@ -1016,3 +1016,54 @@ def test_bootstrap_binary_disabled(self):
         self.assert_log_had_msg(node3, "Leaving write survey mode and joining ring at operator request")
         assert_bootstrap_state(self, node3, 'COMPLETED', user='cassandra', password='cassandra')
         node3.wait_for_binary_interface()
+
+    @since('4.0')
+    @pytest.mark.no_vnodes
+    def test_simple_bootstrap_with_everywhere_strategy(self):
+        cluster = self.cluster
+        tokens = cluster.balanced_tokens(2)
+        cluster.set_configuration_options(values={'num_tokens': 1})
+
+        logger.debug("[node1, node2] tokens: %r" % (tokens,))
+
+        keys = 10000
+
+        # Create a single node cluster
+        cluster.populate(1)
+        node1 = cluster.nodelist()[0]
+        node1.set_configuration_options(values={'initial_token': tokens[0]})
+        cluster.start()
+
+        session = self.patient_cql_connection(node1)
+        create_ks(session, 'ks', 'EverywhereStrategy')
+        create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
+
+        insert_statement = session.prepare("INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')")
+        execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)])
+
+        node1.flush()
+        node1.compact()
+
+        # Reads inserted data all during the bootstrap process. We shouldn't
+        # get any error
+        query_c1c2(session, random.randint(0, keys - 1), ConsistencyLevel.ONE)
+        session.shutdown()
+
+        # Bootstrapping a new node in the current version
+        node2 = new_node(cluster)
+        node2.set_configuration_options(values={'initial_token': tokens[1]})
+        node2.start(wait_for_binary_proto=True)
+        node2.compact()
+
+        node1.cleanup()
+        logger.debug("node1 size for ks.cf after cleanup: %s" % float(data_size(node1,'ks','cf')))
+        node1.compact()
+        logger.debug("node1 size for ks.cf after compacting: %s" % float(data_size(node1,'ks','cf')))
+
+        logger.debug("node2 size for ks.cf after compacting: %s" % float(data_size(node2,'ks','cf')))
+
+        size1 = float(data_size(node1,'ks','cf'))
+        size2 = float(data_size(node2,'ks','cf'))
+        assert_almost_equal(size1, size2, error=0.3)
+
+        assert_bootstrap_state(self, node2, 'COMPLETED')

conftest.py

@@ -43,6 +43,9 @@ def check_required_loopback_interfaces_available():
 
 
 def pytest_addoption(parser):
+    parser.addoption("--sstable-format", action="store", default="bti",
+                     help="SSTable format to be used by default for all newly created SSTables: "
+                          "big or bti (default: bti)")
     parser.addoption("--use-vnodes", action="store_true", default=False,
                      help="Determines wither or not to setup clusters using vnodes for tests")
     parser.addoption("--use-off-heap-memtables", action="store_true", default=False,

cqlsh_tests/test_cqlsh.py

@@ -1080,6 +1080,7 @@ def get_test_table_output(self, has_val=True, has_val_idx=True):
             AND comment = ''
             AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
             AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+            AND memtable = {}
             AND crc_check_chance = 1.0
             AND default_time_to_live = 0
             AND extensions = {}
@@ -1169,6 +1170,7 @@ def get_users_table_output(self):
             AND comment = ''
             AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
             AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+            AND memtable = {}
             AND crc_check_chance = 1.0
             AND default_time_to_live = 0
             AND extensions = {}
@@ -1275,6 +1277,7 @@ def get_users_by_state_mv_output(self):
                 AND comment = ''
                 AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
                 AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+                AND memtable = {}
                 AND crc_check_chance = 1.0
                 AND default_time_to_live = 0
                 AND extensions = {}

cqlsh_tests/test_cqlsh_copy.py

@@ -3330,3 +3330,79 @@ def _test_invalid_data_for_maps():
         _test_invalid_data_for_sets()
         _test_invalid_data_for_lists()
         _test_invalid_data_for_maps()
+
+    @since('4.0')
+    def test_geotypes_copy(self):
+        """
+        Tests whether cqlsh COPY properly handles geo types with empty and null values.
+
+        @since 4.0.0
+
+        Steps:
+        * insert several geoTypes with null and empty values among them into a table
+        * cqlsh copy contents to .csv file and save them in a list
+        * wipe the table comletely of all data
+        * cqlsh copy contents from .csv back into the table
+        * get the contents of the table into a list
+        * assert the pre wiped data is the same as the newly copied data
+        :return
+        """
+        self.prepare()
+
+        self.session.execute("create table geo (k int primary key, point 'PointType', line 'LineStringType', poly 'PolygonType');")
+        self.session.execute("insert into geo (k, point, line, poly) VALUES (0, 'point(1.2 3.4)', 'linestring(1.0 1.1, 2.0 2.1, 3.0 3.1)', 'POLYGON ((10.1 10.0, 110.0 10.0, 110.0 110.0, 10.0 110.0, 10.0 10.0), (20.0 20.0, 20.0 30.0, 30.0 30.0, 30.0 20.0, 20.0 20.0))');")
+        self.session.execute("insert into geo (k, point, line, poly) VALUES (2, 'point(1.2 3.4)', 'linestring EMPTY', 'POLYGON EMPTY');")
+        self.session.execute("insert into geo (k) VALUES (1);")
+
+        # make sure data is inserted
+        data_actual = rows_to_list(self.session.execute("select * from geo;"))
+        assert len(data_actual) == 3
+
+        # dump data to CSV and truncate
+        tempfile = self.get_temp_file()
+        self.run_cqlsh(cmds="COPY ks.geo TO '{name}'".format(name=tempfile.name))
+        self.run_cqlsh(cmds="truncate ks.geo;")
+
+        # import data back
+        self.run_cqlsh(cmds="COPY ks.geo FROM '{name}'".format(name=tempfile.name))
+        data_copy = rows_to_list(self.session.execute("select * from geo;"))
+
+        assert data_actual == data_copy
+
+    @since("4.0")
+    def test_date_range_copy(self):
+        """
+        DateRangeTests.test_copy_command
+
+        Tests whether cqlsh COPY properly handles date_range types, including null values.
+        @note we cannot insert empty value ('') as it is not presented as null in cqlsh but it is in COPY
+        """
+        self.prepare()
+
+        self.session.execute("create table incomes (org text, period 'DateRangeType', incomes int, ver 'DateRangeType', primary key (org, period));")
+        # insert some data
+        self.session.execute("insert into incomes(org, period, incomes) values ('A','2014', 20140);")
+        self.session.execute("insert into incomes(org, period, incomes) values ('A','2015', 201500);")
+        self.session.execute("insert into incomes(org, period, incomes) values ('A','[2016-01-01 TO 2016-06-30]', 1007);")
+        self.session.execute("insert into incomes(org, period, incomes) values ('B','[2017-02-12 12:30:07 TO 2017-02-17 13:39:43.789]', 777);")
+        self.session.execute("insert into incomes(org, period, incomes, ver) values ('X','2011', 0, null);")
+        self.session.execute("insert into incomes(org, period, incomes) values ('C','*', 996);")
+        self.session.execute("insert into incomes(org, period, incomes) values ('C','[* TO *]', 997);")
+        self.session.execute("insert into incomes(org, period, incomes) values ('C','[* TO 2015-01]', 998);")
+        self.session.execute("insert into incomes(org, period, incomes) values ('C','[2015-01 TO *]', 999);")
+
+        # make sure data is inserted
+        data_actual = rows_to_list(self.session.execute("select * from incomes;"))
+        assert len(data_actual) == 9
+
+        # dump data to CSV and truncate
+        tempfile = self.get_temp_file()
+        self.run_cqlsh(cmds="COPY ks.incomes TO '{name}'".format(name=tempfile.name))
+        self.run_cqlsh(cmds="truncate ks.incomes;")
+
+        # import data back
+        self.run_cqlsh(cmds="COPY ks.incomes FROM '{name}'".format(name=tempfile.name))
+        data_copy = rows_to_list(self.session.execute("select * from incomes;"))
+
+        assert data_actual == data_copy
+

cqlsh_tests/test_cqlsh_types.py

@@ -0,0 +1,67 @@
+import logging
+import pytest
+
+from dtest import Tester, create_ks
+
+logger = logging.getLogger(__name__)
+since = pytest.mark.since
+
+
+@since("4.0")
+class TestCqlshTypes(Tester):
+
+    def prepare(self, workload=None):
+        if not self.cluster.nodelist():
+            self.allow_log_errors = True
+            self.cluster.populate(1)
+            if workload is not None:
+                for node in self.cluster.nodelist():
+                    node.set_workload(workload)
+            logger.debug('About to start cluster')
+            self.cluster.start()
+            logger.debug('Cluster started')
+            for node in self.cluster.nodelist():
+                node.watch_log_for('Starting listening for CQL clients', timeout=60)
+            self.cluster.nodelist()[0].watch_log_for('Created default superuser')
+            self.node = self.cluster.nodelist()[0]
+
+            conn = self.patient_cql_connection(self.node)
+            create_ks(conn, 'ks', 1)
+
+        logger.debug('prepare completed')
+
+    def test_point(self):
+        self.prepare()
+
+        expected = 'POINT (1.2 2.3)'
+        self.node.run_cqlsh("CREATE TABLE ks.point_tbl (k INT PRIMARY KEY, point 'PointType');")
+        self.node.run_cqlsh("INSERT INTO ks.point_tbl (k, point) VALUES (1, '{}')".format(expected))
+        result = self.node.run_cqlsh("SELECT * FROM ks.point_tbl;")
+        assert expected in result[0], result
+
+    def test_linestring(self):
+        self.prepare()
+
+        expected = 'LINESTRING (30.0 10.0, 10.0 30.0, 40.0 40.0)'
+        self.node.run_cqlsh("CREATE TABLE ks.line_tbl (k INT PRIMARY KEY, linestring 'LineStringType');")
+        self.node.run_cqlsh("INSERT INTO ks.line_tbl (k, linestring) VALUES (1, '{}')".format(expected))
+        result = self.node.run_cqlsh("SELECT * FROM ks.line_tbl;")
+        assert expected in result[0], result
+
+    def test_polygon(self):
+        self.prepare()
+
+        expected = 'POLYGON ((30.0 10.0, 40.0 40.0, 20.0 40.0, 10.0 20.0, 30.0 10.0))'
+        self.node.run_cqlsh("CREATE TABLE ks.polygon_tbl (k INT PRIMARY KEY, polygon 'PolygonType');")
+        self.node.run_cqlsh("INSERT INTO ks.polygon_tbl (k, polygon) VALUES (1, '{}')".format(expected))
+        result = self.node.run_cqlsh("SELECT * FROM ks.polygon_tbl;")
+        assert expected in result[0], result
+
+    def test_date_range(self):
+        self.prepare()
+
+        expected = '[2015-01 TO *]'
+        self.node.run_cqlsh("CREATE TABLE ks.date_range_tbl (k INT PRIMARY KEY, date_range_tbl 'DateRangeType');")
+        self.node.run_cqlsh("INSERT INTO ks.date_range_tbl (k, date_range_tbl) VALUES (1, '{}')".format(expected))
+        result = self.node.run_cqlsh("SELECT * FROM ks.date_range_tbl;")
+        assert expected in result[0], result

dtest.py

@@ -359,6 +359,8 @@ def create_ks(session, name, rf):
     if isinstance(rf, int):
         # we assume simpleStrategy
         query = query % (name, "'class':'SimpleStrategy', 'replication_factor':%d" % rf)
+    elif 'EverywhereStrategy' in rf:
+        query = query % (name, "'class':'org.apache.cassandra.locator.EverywhereStrategy'")
     else:
         assert len(rf) >= 0, "At least one datacenter/rf pair is needed"
         # we assume networkTopologyStrategy

dtest_config.py

@@ -11,6 +11,7 @@
 
 class DTestConfig:
     def __init__(self):
+        self.sstable_format = "bti"
         self.use_vnodes = True
         self.use_off_heap_memtables = False
         self.num_tokens = -1
@@ -41,6 +42,12 @@ def setup(self, config):
             self.cassandra_version_from_build = self.get_version_from_build()
             return
 
+        self.sstable_format = config.getoption("--sstable-format")
+        if self.sstable_format:
+            assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format)
+            default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format
+            os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop})
+
         self.use_vnodes = config.getoption("--use-vnodes")
         self.use_off_heap_memtables = config.getoption("--use-off-heap-memtables")
         self.num_tokens = config.getoption("--num-tokens")

offline_tools_test.py

@@ -266,9 +266,9 @@ def test_sstableverify(self):
             hashcomputed = False
             for line in outlines:
                 if sstable in line:
-                    if "Verifying BigTableReader" in line:
+                    if "Verifying " in line:
                         verified = True
-                    elif "Checking computed hash of BigTableReader" in line:
+                    elif "Checking computed hash of " in line:
                         hashcomputed = True
                     else:
                         logger.debug(line)

run_dtests.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python
 """
-usage: run_dtests.py [-h] [--use-vnodes] [--use-off-heap-memtables] [--num-tokens=NUM_TOKENS] [--data-dir-count-per-instance=DATA_DIR_COUNT_PER_INSTANCE]
+usage: run_dtests.py [-h] [--sstable-format=FORMAT] [--use-vnodes] [--use-off-heap-memtables] [--num-tokens=NUM_TOKENS] [--data-dir-count-per-instance=DATA_DIR_COUNT_PER_INSTANCE]
                      [--force-resource-intensive-tests] [--skip-resource-intensive-tests] [--cassandra-dir=CASSANDRA_DIR] [--cassandra-version=CASSANDRA_VERSION]
                      [--delete-logs] [--execute-upgrade-tests] [--execute-upgrade-tests-only] [--disable-active-log-watching] [--keep-test-dir]
                      [--enable-jacoco-code-coverage] [--dtest-enable-debug-logging] [--dtest-print-tests-only] [--dtest-print-tests-output=DTEST_PRINT_TESTS_OUTPUT]
                      [--pytest-options=PYTEST_OPTIONS] [--dtest-tests=DTEST_TESTS]
 
 optional arguments:
   -h, --help                                                 show this help message and exit
+  --sstable-format                                           SSTable format to be used by default for all newly created SSTables: big or bti (default: bti)
   --use-vnodes                                               Determines wither or not to setup clusters using vnodes for tests (default: False)
   --use-off-heap-memtables                                   Enable Off Heap Memtables when creating test clusters for tests (default: False)
   --num-tokens=NUM_TOKENS                                    Number of tokens to set num_tokens yaml setting to when creating instances with vnodes enabled (default: 256)

update-history/STAR-617-2/5-8c968011 STAR-247: Allow to easily run tests against big or bti sstable format

@@ -0,0 +1,27 @@
+commit 8c9680112dc2ba92b04460e695b74fa0273b2791
+Author:     jacek-lewandowski <[email protected]>
+AuthorDate: Fri Apr 2 13:51:45 2021 +0200
+Commit:     jacek-lewandowski <[email protected]>
+CommitDate: Mon Apr 12 17:01:32 2021 +0200
+
+    STAR-247: Allow to easily run tests against big or bti sstable format
+
+Resolutions:
+diff --git a/dtest_config.py b/dtest_config.py
+index fbe15a9f..116566b2 100644
+--- a/dtest_config.py
++++ b/dtest_config.py
+@@ -43,9 +43,10 @@ class DTestConfig:
+             return
+
+         self.sstable_format = config.getoption("--sstable-format")
+-        assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format)
+-        default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format
+-        os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop})
++        if self.sstable_format:
++            assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format)
++            default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format
++            os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop})
+
+         self.use_vnodes = config.getoption("--use-vnodes")
+         self.use_off_heap_memtables = config.getoption("--use-off-heap-memtables")