1

scylladb · Jan 12, 2025 · d8e6b6d · d8e6b6d
1 parent c8f6fd9
commit d8e6b6d
Show file tree

Hide file tree

Showing 14 changed files with 1,693 additions and 402 deletions.
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -9,7 +9,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -18,38 +18,35 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Cache pip
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-test.txt') }}
         restore-keys: |
           ${{ runner.os }}-pip-
 
     - name: Setup java
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: '8'
 
     - name: Install dependencies
       run: |
-        sudo apt-get update
-        sudo apt-get install python2
-        
+        pip install --upgrade pip
         pip install -U pip setuptools
-
         pip install .
 
         if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi
 
     - name: Cache binary versions
       id: cache-versions
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: |
           ~/.ccm/repository

diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml
@@ -19,7 +19,7 @@ jobs:
 
     - name: Cache binary versions
       id: cache-versions
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: |
           ~/.ccm/repository

diff --git a/ccmlib/common.py b/ccmlib/common.py
diff --git a/ccmlib/node.py b/ccmlib/node.py
@@ -257,12 +257,12 @@ def set_workload(self, workload):
 
     def get_cassandra_version(self):
         try:
-            return common.get_version_from_build(self.get_install_dir())
+            return str(common.get_version_from_build(self.get_install_dir()))
         except common.CCMError:
             return self.cluster.cassandra_version()
 
     def get_base_cassandra_version(self):
-        version = self.get_cassandra_version()
+        version = str(self.get_cassandra_version())
         return float(version[:version.index('.') + 2])
 
     def set_configuration_options(self, values=None, batch_commitlog=None):

diff --git a/ccmlib/repository.py b/ccmlib/repository.py
diff --git a/requirements-test.txt b/requirements-test.txt
@@ -1,2 +1,4 @@
+-r requirements.txt
 pytest
-awscli
+awscli
+mock
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,7 @@
+ruamel.yaml
+psutil
+requests
+packaging
+boto3
+tqdm
+urllib3<2
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -1,5 +1,3 @@
-import os
-import shutil
 import sys
 
 
@@ -8,16 +6,3 @@
 
 if sys.version_info < (3, 0):
     FileNotFoundError = OSError
-
-
-def setup_package():
-    try:
-        shutil.rmtree(TEST_DIR)
-    except FileNotFoundError:
-        pass
-
-    os.makedirs(TEST_DIR)
-
-
-def teardown_package():
-    shutil.rmtree(TEST_DIR)
diff --git a/tests/ccmtest.py b/tests/ccmtest.py
@@ -0,0 +1,32 @@
+import os
+from unittest import TestCase
+
+try:
+    FileNotFoundError
+except NameError:
+    FileNotFoundError = IOError  # Python 2.7 compatibility
+
+class Tester(TestCase):
+
+    check_log_errors = True
+
+    def __init__(self, *argv, **kwargs):
+        super(Tester, self).__init__(*argv, **kwargs)
+
+    def setUp(self):
+        self.check_log_errors = True
+
+    def tearDown(self):
+        if hasattr(self, 'cluster'):
+            try:
+                if self.check_log_errors:
+                    for node in self.cluster.nodelist():
+                        try:
+                            self.assertListEqual(node.grep_log_for_errors(), [])
+                        except FileNotFoundError:
+                            continue
+            finally:
+                test_path = self.cluster.get_path()
+                self.cluster.remove()
+                if os.path.exists(test_path):
+                    os.remove(test_path)
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -5,11 +5,12 @@
 
 import pytest
 from tests.test_config import RESULTS_DIR, TEST_ID, SCYLLA_DOCKER_IMAGE, SCYLLA_RELOCATABLE_VERSION
+import shutil
 
 from ccmlib.scylla_cluster import ScyllaCluster
 from ccmlib.scylla_docker_cluster import ScyllaDockerCluster
 from .ccmcluster import CCMCluster
-
+from tests import TEST_DIR
 
 LOGGER = logging.getLogger(__name__)
 
@@ -60,10 +61,6 @@ def docker_cluster(test_dir, test_id):
     })
     cluster.populate(3)
     cluster.start(wait_for_binary_proto=True)
-    try:
-        yield cluster
-    finally:
-        cluster.clear()
 
 
 @pytest.fixture(scope="session")
@@ -88,6 +85,18 @@ def relocatable_cluster(test_dir, test_id):
         cluster.clear()
 
 
+@pytest.fixture(scope="package", autouse=True)
+def setup_and_package():
+    try:
+        shutil.rmtree(TEST_DIR)
+    except FileNotFoundError:
+        pass
+
+    os.makedirs(TEST_DIR)
+    yield
+    shutil.rmtree(TEST_DIR)
+
+
 @pytest.fixture(scope="session")
 def ccm_docker_cluster():
     cluster = CCMCluster(test_id="docker", docker_image=SCYLLA_DOCKER_IMAGE)

diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -0,0 +1,4 @@
+mock==5.1.0
+pytest==8.2.1
+requests==2.32.2
+pylint==3.2.2
diff --git a/tests/test_common.py b/tests/test_common.py
@@ -1,98 +1,100 @@
-import tempfile
-import os
-
-import pytest
-import ruamel
-
-from ccmlib.common import scylla_extract_mode, LockFile, parse_settings
-
-
-def test_scylla_extract_mode():
-    assert scylla_extract_mode("build/dev") == 'dev'
-    assert scylla_extract_mode("../build/release") == 'release'
-    assert scylla_extract_mode("../build/release/scylla") == 'release'
-    assert scylla_extract_mode("/home/foo/scylla/build/debug") == 'debug'
-    assert scylla_extract_mode("url=../scylla/build/debug/scylla-package.tar.gz") == 'debug'
-
-    assert scylla_extract_mode("url=./scylla-debug-x86_64-package.tar.gz") == 'debug'
-    assert scylla_extract_mode("url=./scylla-x86_64-package.tar.gz") == 'release'
-    assert scylla_extract_mode("url=./scylla-debug-aarch64-package.tar.gz") == 'debug'
-    assert scylla_extract_mode("url=./scylla-package.tar.gz") == 'release'
-    assert scylla_extract_mode("url=./scylla-debug-package.tar.gz") == 'debug'
-
-    assert scylla_extract_mode("url=./scylla-debug-x86_64-package-4.5.2.0.20211114.26aca7b9f.tar.gz") == 'debug'
-    assert scylla_extract_mode("url=./scylla-debug-package-4.5.2.0.20211114.26aca7b9f.tar.gz") == 'debug'
-    assert scylla_extract_mode("url=./scylla-package-4.5.2.0.20211114.26aca7b9f.tar.gz") == 'release'
-
-    assert scylla_extract_mode('url=https://s3.amazonaws.com/downloads.scylladb.com/downloads/scylla-enterprise/'
-                               'relocatable/scylladb-2022.1/scylla-enterprise-x86_64-package-2022.1.rc6.0.20220523.'
-                               '30ce52b2e.tar.gz') == 'release'
-    assert scylla_extract_mode('url=https://s3.amazonaws.com/downloads.scylladb.com/downloads/scylla-enterprise/'
-                               'relocatable/scylladb-2022.1/scylla-enterprise-debug-aarch64-package-2022.1.rc0.0.20220331.f3ee71fba.tar.gz') == 'debug'
-
-    assert scylla_extract_mode("url=https://downloads.scylla.com/relocatable/unstable/master/202001192256/scylla-debug-package.tar.gz") == 'debug'
-    assert scylla_extract_mode("url=https://downloads.scylladb.com/unstable/scylla/master/relocatable/latest/scylla-debug-unified-5.4.0~dev-0.20230801.37b548f46365.x86_64.tar.gz") == 'debug'
-    assert scylla_extract_mode("url=https://s3.amazonaws.com/downloads.scylladb.com/unstable/scylla-enterprise/enterprise/relocatable/latest/scylla-enterprise-debug-unstripped-2023.3.0~dev-0.20230806.6dc3aeaf312c.aarch64.tar.gz") == 'debug'
-    assert scylla_extract_mode("/jenkins/workspace/scylla-master/dtest-debug/scylla/build/debug/dist/tar/scylla-debug-unified-5.4.0~dev-0.20231013.055f0617064d.x86_64.tar.gz") == 'debug'
-
-# Those tests assume that LockFile uses fcntl.flock
-# If it switches to anything else, the tests need to be adjusted.
-
-def test_lockfile_basic():
-    f, path = tempfile.mkstemp(prefix='ccm-test-lockfile')
-    lf = LockFile(path)
-    assert lf.acquire(blocking=True) == (True, None)
-
-    assert lf.read_contents() == (os.getpid(), '')
-    lf.write_status('abc')
-    assert lf.read_contents() == (os.getpid(), 'abc')
-
-    lf.release()
-
-
-def test_lockfile_locks():
-    f, path = tempfile.mkstemp(prefix='ccm-test-lockfile')
-    lf1 = LockFile(path)
-    lf2 = LockFile(path)
-    with lf1:
-        assert lf2.acquire(blocking=False) == (False, os.getpid())
-    assert lf2.acquire(blocking=False) == (True, None)
-    assert lf1.acquire(blocking=False) == (False, os.getpid())
-    lf2.release()
-
-
-def test_lockfile_retain_status_by_default():
-    f, path = tempfile.mkstemp(prefix='ccm-test-lockfile')
-    lf = LockFile(path)
-
-    assert lf.acquire(blocking=False)[0] is True
-    lf.write_status('some_status_1')
-    assert lf.read_status() == 'some_status_1'
-    lf.release()
-
-    # Status should be retained from previous lock.
-    assert lf.acquire(blocking=False)[0] is True
-    assert lf.read_status() == 'some_status_1'
-    lf.release()
-
-def test_parse_settings():
-    res = parse_settings(["number:12", "string:somthing", "bool:false"])
-    assert res == {'bool': False, 'number': 12, 'string': 'somthing'}
-
-    res = parse_settings(["nested.number:12", "nested.string:somthing", "nested.bool:false"])
-    assert res == {'nested': {'bool': False, 'number': 12, 'string': 'somthing'}}
-
-    # a fix that need to be backported from upstream
-    # https://github.com/riptano/ccm/commit/37afe6ab86fe03d5be7d0cf7a328dccac035a9a0
-    # res = parse_settings(["double.nested.number:12", "double.nested.string:somthing", "double.nested.bool:false"])
-    # assert res == {'double': {'nested': {'bool': False, 'number': 12, 'string': 'somthing'}}}
-
-    res = parse_settings(["experimental_features:[udf,tablets]"])
-    assert res == {'experimental_features': ['udf', 'tablets']}
-
-    res = parse_settings(["experimental_features:['udf',\"tablets\"]"])
-    assert res == {'experimental_features': ['udf', 'tablets']}
-
-    # would break if incorrect yaml format is passed in the value
-    with pytest.raises(ruamel.yaml.parser.ParserError):
-        parse_settings(["experimental_features:['udf',\"tablets\""])
+import unittest
+from mock import patch
+
+from distutils.version import LooseVersion
+
+from ccmlib import common
+from . import ccmtest
+
+
+class TestCommon(ccmtest.Tester):
+
+    def test_normalize_interface(self):
+        normalized = common.normalize_interface(('::1', 9042))
+        self.assertEqual(normalized, ('0:0:0:0:0:0:0:1', 9042))
+
+        normalized = common.normalize_interface(('127.0.0.1', 9042))
+        self.assertEqual(normalized, ('127.0.0.1', 9042))
+
+        normalized = common.normalize_interface(('fe80::3e15:c2ff:fed3:db74%en0', 9042))
+        self.assertEqual(normalized, ('fe80:0:0:0:3e15:c2ff:fed3:db74%en0', 9042))
+
+        normalized = common.normalize_interface(('fe80::1%lo0', 9042))
+        self.assertEqual(normalized, ('fe80:0:0:0:0:0:0:1%lo0', 9042))
+
+        normalized = common.normalize_interface(('fd6d:404d:54cb::1', 9042))
+        self.assertEqual(normalized, ('fd6d:404d:54cb:0:0:0:0:1', 9042))
+
+    @patch('ccmlib.common.is_win')
+    def test_is_modern_windows_install(self, mock_is_win):
+        mock_is_win.return_value = True
+        self.assertTrue(common.is_modern_windows_install(2.1))
+        self.assertTrue(common.is_modern_windows_install('2.1'))
+        self.assertTrue(common.is_modern_windows_install(LooseVersion('2.1')))
+
+        self.assertTrue(common.is_modern_windows_install(3.12))
+        self.assertTrue(common.is_modern_windows_install('3.12'))
+        self.assertTrue(common.is_modern_windows_install(LooseVersion('3.12')))
+
+        self.assertFalse(common.is_modern_windows_install(1.0))
+        self.assertFalse(common.is_modern_windows_install('1.0'))
+        self.assertFalse(common.is_modern_windows_install(LooseVersion('1.0')))
+
+    def test_merge_configuration(self):
+        # test for merging dict val in key, value pair
+        dict0 = dict1 = {'key': {'val1': True}}
+        dict2 = {'key': {'val2': False}}
+
+        for k, v in dict2.items():
+            dict0[k].update(v)
+
+        self.assertEqual(common.merge_configuration(dict1, dict2), dict0)
+
+        # test for merging str val in key, value pair
+        dict0 = dict1 = {'key': 'val1'}
+        dict2 = {'key': 'val2'}
+
+        for k, v in dict2.items():
+            dict0[k] = v
+
+        self.assertEqual(common.merge_configuration(dict1, dict2), dict0)
+
+    def test_get_jdk_version(self):
+        v8u152 = """java version "1.8.0_152"
+                 Java(TM) SE Runtime Environment (build 1.8.0_152-b16)
+                 Java HotSpot(TM) 64-Bit Server VM (build 25.152-b16, mixed mode)
+                 """
+        # Since Java 9, the version string syntax changed.
+        # Most relevant change is that trailing .0's are omitted. I.e. Java "9.0.0"
+        # version string is not "9.0.0" but just "9".
+        v900 = """java version "9"
+               Java(TM) SE Runtime Environment (build 9+1)
+               Java HotSpot(TM) 64-Bit Server VM (build 9+1, mixed mode)
+               """
+        v901 = """java version "9.0.1"
+               Java(TM) SE Runtime Environment (build 9.0.1+11)
+               Java HotSpot(TM) 64-Bit Server VM (build 9.0.1+11, mixed mode)
+               """
+        # 10-internal, just to have an internal (local) build in here
+        v10_int = """openjdk version "10-internal"
+                  OpenJDK Runtime Environment (build 10-internal+0-adhoc.jenkins.openjdk-shenandoah-jdk10-release)
+                  OpenJDK 64-Bit Server VM (build 10-internal+0-adhoc.jenkins.openjdk-shenandoah-jdk10-release, mixed mode)
+                  """
+        v1000 = """java version "10"
+                Java(TM) SE Runtime Environment (build 9+1)
+                Java HotSpot(TM) 64-Bit Server VM (build 9+1, mixed mode)
+                """
+        v1001 = """java version "10.0.1"
+                Java(TM) SE Runtime Environment (build 10.0.1+11)
+                Java HotSpot(TM) 64-Bit Server VM (build 10.0.1+11, mixed mode)
+                """
+
+        self.assertEqual(common._get_jdk_version(v8u152), "1.8")
+        self.assertEqual(common._get_jdk_version(v900), "9.0")
+        self.assertEqual(common._get_jdk_version(v901), "9.0")
+        self.assertEqual(common._get_jdk_version(v10_int), "10.0")
+        self.assertEqual(common._get_jdk_version(v1000), "10.0")
+        self.assertEqual(common._get_jdk_version(v1001), "10.0")
+
+if __name__ == '__main__':
+    unittest.main()