Add sni-proxy support to CCM

So the different drivers can be test their new
implementations of sni-proxy

for using it, need to start the cluster from the commandline
like this:
```
❯ ccm start --sni-proxy
sni_proxy listening on: 127.0.0.1:443
```

using it from python code, would be a bit diffrent:

```python
nodes_info = get_cluster_info(self.cluster.get_path(),
                              address=self.cluster.nodelist()[0].address(),
                              port=9142)
docker_id, listen_address, listen_port = \
  start_sni_proxy(self.cluster.get_path(), nodes_info=nodes_info)
```

Ref: scylladb/gocql#97

MANIFEST.in

@@ -1,4 +1,5 @@
 include *.md
 include ccmlib/resources/bin/*.sh
 include ccmlib/scylla_test_ssl/scylla-manager-agent.key
-include ccmlib/scylla_test_ssl/scylla-manager-agent.crt
+include ccmlib/scylla_test_ssl/scylla-manager-agent.crt
+include ccmlib/resources/docker/sniproxy/*

ccmlib/cluster.py

@@ -615,8 +615,8 @@ def _update_config(self, install_dir=None):
         node_list = [node.name for node in list(self.nodes.values())]
         seed_list = [node.name for node in self.seeds]
         filename = os.path.join(self.path, self.name, 'cluster.conf')
-        with open(filename, 'w') as f:
-            yaml.safe_dump({
+
+        cluster_config = {
                 'name': self.name,
                 'nodes': node_list,
                 'seeds': seed_list,
@@ -628,7 +628,12 @@ def _update_config(self, install_dir=None):
                 'use_vnodes': self.use_vnodes,
                 'id': self.id,
                 'ipprefix': self.ipprefix
-            }, f)
+            }
+        if getattr(self, 'sni_proxy_docker_id', None):
+            cluster_config['sni_proxy_docker_id'] = self.sni_proxy_docker_id
+
+        with open(filename, 'w') as f:
+            yaml.safe_dump(cluster_config, f)
 
     def __update_pids(self, started):
         for node, p, _ in started:

ccmlib/cluster_factory.py

@@ -48,6 +48,9 @@ def load(path, name):
                 cluster.__log_level = data['log_level']
             if 'use_vnodes' in data:
                 cluster.use_vnodes = data['use_vnodes']
+            if 'sni_proxy_docker_id' in data and data['sni_proxy_docker_id']:
+                cluster.sni_proxy_docker_id = data['sni_proxy_docker_id']
+
         except KeyError as k:
             raise common.LoadError("Error Loading " + filename + ", missing property:" + str(k))
 

ccmlib/cmds/cluster_cmds.py

@@ -1,6 +1,7 @@
 import os
 import subprocess
 import sys
+import tempfile
 
 from six import print_
 
@@ -15,6 +16,8 @@
 from ccmlib.scylla_node import ScyllaNode
 from ccmlib.dse_node import DseNode
 from ccmlib.node import Node, NodeError
+from ccmlib.utils.ssl_utils import generate_ssl_stores
+from ccmlib.utils.sni_proxy import get_cluster_info, start_sni_proxy, refresh_certs
 
 os.environ['SCYLLA_CCM_STANDALONE'] = '1'
 
@@ -628,6 +631,9 @@ def get_parser(self):
         parser.add_option('--profile-opts', type="string", action="store", dest="profile_options",
                           help="Yourkit options when profiling", default=None)
         parser.add_option('--quiet-windows', action="store_true", dest="quiet_start", help="Pass -q on Windows 2.2.4+ and 3.0+ startup. Ignored on linux.", default=False)
+
+        parser.add_option('--sni-proxy', action="store_true", dest="sni_proxy", help="Start sniproxy infront of the cluster", default=False)
+
         return parser
 
     def validate(self, parser, options, args):
@@ -644,7 +650,17 @@ def run(self):
             if len(self.cluster.nodes) == 0:
                 print_("No node in this cluster yet. Use the populate command before starting.")
                 sys.exit(1)
-
+            if self.options.sni_proxy:
+                generate_ssl_stores(self.cluster.get_path())
+                self.cluster.set_configuration_options(dict(
+                        client_encryption_options=
+                            dict(require_client_auth=True,
+                                 truststore=os.path.join(self.cluster.get_path(), 'ccm_node.cer'),
+                                 certificate=os.path.join(self.cluster.get_path(), 'ccm_node.pem'),
+                                 keyfile=os.path.join(self.cluster.get_path(), 'ccm_node.key'),
+                                 enabled=True),
+                        native_transport_port_ssl=9142))
+                self.options.wait_for_binary_proto = True
             if self.cluster.start(no_wait=self.options.no_wait,
                                   wait_other_notice=self.options.wait_other_notice,
                                   wait_for_binary_proto=self.options.wait_for_binary_proto,
@@ -657,6 +673,16 @@ def run(self):
                     details = " (you can use --verbose for more information)"
                 print_("Error starting nodes, see above for details%s" % details, file=sys.stderr)
                 sys.exit(1)
+            if self.options.sni_proxy:
+                nodes_info = get_cluster_info(self.cluster,
+                                              port=9142)
+                refresh_certs(self.cluster, nodes_info)
+                docker_id, listen_address, listen_port = \
+                    start_sni_proxy(self.cluster.get_path(), nodes_info=nodes_info)
+                print('sni_proxy listening on: {}:{}'.format(listen_address, listen_port))
+                self.cluster.sni_proxy_docker_id = docker_id
+                self.cluster._update_config()
+
         except NodeError as e:
             print_(str(e), file=sys.stderr)
             if e.process is not None:

ccmlib/cmds/node_cmds.py

@@ -7,6 +7,7 @@
 from ccmlib import common
 from ccmlib.cmds.command import Cmd
 from ccmlib.node import NodeError
+from ccmlib.utils.sni_proxy import get_cluster_info, refresh_certs, start_sni_proxy, stop_sni_proxy
 
 
 def node_cmds():
@@ -199,6 +200,17 @@ def run(self):
                             replace_address=self.options.replace_address,
                             jvm_args=self.options.jvm_args,
                             quiet_start=self.options.quiet_start)
+
+            if getattr(self.cluster, 'sni_proxy_docker_id', None):
+                nodes_info = get_cluster_info(self.cluster, port=9142)
+                refresh_certs(self.cluster, nodes_info)
+                stop_sni_proxy(self.cluster.sni_proxy_docker_id)
+                docker_id, listen_address, listen_port = \
+                    start_sni_proxy(self.cluster.get_path(), nodes_info=nodes_info)
+                print('sni_proxy listening on: {}:{}'.format(listen_address, listen_port))
+                self.cluster.sni_proxy_docker_id = docker_id
+                self.cluster._update_config()
+
         except NodeError as e:
             print_(str(e), file=sys.stderr)
             print_("Standard error output is:", file=sys.stderr)

ccmlib/resources/docker/sniproxy/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine
+
+RUN apk --no-cache add sniproxy
+
+EXPOSE 80 443
+ENTRYPOINT ["sniproxy", "-f"]
+

ccmlib/scylla_cluster.py

@@ -16,6 +16,7 @@
 from ccmlib.scylla_node import ScyllaNode
 from ccmlib.node import NodeError
 from ccmlib import scylla_repository
+from ccmlib.utils.sni_proxy import stop_sni_proxy
 
 SNITCH = 'org.apache.cassandra.locator.GossipingPropertyFileSnitch'
 
@@ -186,6 +187,10 @@ def stop_nodes(self, nodes=None, wait=True, gently=True, wait_other_notice=False
         return [node for node in nodes if not node.is_running()]
 
     def stop(self, wait=True, gently=True, wait_other_notice=False, other_nodes=None, wait_seconds=None):
+        if getattr(self, 'sni_proxy_docker_id', None):
+            stop_sni_proxy(self.sni_proxy_docker_id)
+            self.sni_proxy_docker_id = None
+
         if self._scylla_manager and not self.skip_manager_server:
             self._scylla_manager.stop(gently)
         kwargs = dict(**locals())

ccmlib/utils/sni_proxy.py

@@ -0,0 +1,169 @@
+import os
+import string
+import subprocess
+import json
+import base64
+from contextlib import contextmanager
+import tempfile
+from textwrap import dedent
+import distutils.dir_util
+
+import yaml
+
+from ccmlib.utils.ssl_utils import generate_ssl_stores
+
+
+@contextmanager
+def file_or_memory(path=None, data=None):
+    # since we can't read keys/cert from memory yet
+    # see https://github.com/python/cpython/pull/2449 which isn't accepted and PEP-543 that was withdrawn
+    # so we use temporary file to load the key
+    if data:
+        with tempfile.NamedTemporaryFile(mode="wb") as f:
+            d = base64.decodebytes(bytes(data, encoding='utf-8'))
+            f.write(d)
+            if not d.endswith(b"\n"):
+                f.write(b"\n")
+
+            f.flush()
+            yield f.name
+
+    if path:
+        yield path
+
+
+def create_cloud_config(ssl_dir, host, port, username='cassandra', password='cassandra'):
+
+    def encode_base64(filename):
+        return base64.b64encode(open(os.path.join(ssl_dir, filename), 'rb').read()).decode()
+
+    cadata = encode_base64('ccm_node.cer')
+    certificate_data = encode_base64('ccm_node.cer')
+    key_data = encode_base64('ccm_node.key')
+
+    config = dict(datacenters={'eu-west-1': dict(certificateAuthorityData=cadata,
+                                                 server=f'{host}:{port}',
+                                                 nodeDomain='cluster-id.scylla.com')},
+                  authInfos={'default': dict(clientCertificateData=certificate_data,
+                                             clientKeyData=key_data,
+                                             username=username,
+                                             password=password,
+                                             insecureSkipTlsVerify=False)},
+                  contexts={'default': dict(datacenterName='eu-west-1', authInfoName='default')},
+                  currentContext='default')
+
+    with open(os.path.join(ssl_dir, 'config_data.yaml'), 'w') as config_file:
+        config_file.write(yaml.safe_dump(config, sort_keys=False))
+
+    config = dict(datacenters={'eu-west-1': dict(certificateAuthorityPath=os.path.join(ssl_dir, 'ccm_node.cer'),
+                                                 server=f'{host}:{port}',
+                                                 nodeDomain='cluster-id.scylla.com')},
+                  authInfos={'default': dict(clientCertificatePath=os.path.join(ssl_dir, 'ccm_node.cer'),
+                                             clientKeyPath=os.path.join(ssl_dir, 'ccm_node.key'),
+                                             username=username,
+                                             password=password,
+                                             insecureSkipTlsVerify=False)},
+                  contexts={'default': dict(datacenterName='eu-west-1', authInfoName='default')},
+                  currentContext='default')
+
+    with open(os.path.join(ssl_dir, 'config_path.yaml'), 'w') as config_file:
+        config_file.write(yaml.safe_dump(config, sort_keys=False))
+
+    return os.path.join(ssl_dir, 'config_data.yaml'), os.path.join(ssl_dir, 'config_path.yaml')
+
+
+def stop_sni_proxy(docker_id):
+    subprocess.check_output(['/bin/bash', '-c', f'docker rm -f {docker_id}'])
+
+
+def configure_sni_proxy(conf_dir, nodes_info, listen_port=443):
+    sniproxy_conf_tmpl = dedent("""
+        user sniproxy
+        pidfile /var/run/sniproxy/sniproxy.pid
+
+        error_log {
+          filename /dev/stderr
+          priority debug
+        }
+
+        listener $FIRST_ADDRESS $listen_port {
+          proto tls
+          access_log {
+            filename /dev/stdout
+          }
+        }
+
+        table {
+        $TABLES
+        }
+        """)
+    tables = ""
+    mapping = {}
+    address, port, host_id = list(nodes_info)[0]
+    tables += f"  any.cluster-id.scylla.com {address}:{port}\n"
+    mapping['FIRST_ADDRESS'] = address
+    mapping['listen_port'] = listen_port
+
+    for address, port, host_id in nodes_info:
+        tables += f"  {host_id}.cluster-id.scylla.com {address}:{port}\n"
+
+    tmpl = string.Template(sniproxy_conf_tmpl)
+    sniproxy_conf_path = os.path.join(conf_dir, 'sniproxy.conf')
+
+    with open(sniproxy_conf_path, 'w') as fp:
+        fp.write(tmpl.substitute(TABLES=tables, **mapping))
+
+    return sniproxy_conf_path
+
+
+def start_sni_proxy(conf_dir, nodes_info, listen_port=443):
+    address, _, _ = list(nodes_info)[0]
+    sniproxy_conf_path = configure_sni_proxy(conf_dir, nodes_info, listen_port=443)
+    sniproxy_dockerfile = os.path.join(os.path.dirname(__file__), '..', 'resources', 'docker', 'sniproxy')
+    subprocess.check_output(['/bin/bash', '-c', f'docker build {sniproxy_dockerfile} -t sniproxy'], universal_newlines=True)
+    docker_id = subprocess.check_output(['/bin/bash', '-c', f'docker run -d --network=host -v {sniproxy_conf_path}:/etc/sniproxy.conf:z -p 443 -it sniproxy'], universal_newlines=True)
+
+    return docker_id.strip(), address, listen_port
+
+
+def get_cluster_info(cluster, port=9142):
+
+    node1 = cluster.nodelist()[0]
+    stdout, stderr = node1.run_cqlsh(cmds='select JSON host_id,broadcast_address from system.local ;',
+                                     return_output=True)
+
+    nodes_info = []
+    for line in stdout.splitlines()[3:-2]:
+        host = json.loads(line)
+        nodes_info.append((host['broadcast_address'], port, host['host_id']))
+
+    stdout, stderr = node1.run_cqlsh(cmds='select JSON peer,host_id from system.peers ;',
+                                     return_output=True)
+
+    for line in stdout.splitlines()[3:-2]:
+        host = json.loads(line)
+        nodes_info.append((host['peer'], port, host['host_id']))
+
+    return nodes_info
+
+
+def refresh_certs(cluster, nodes_info):
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        dns_names = ['any.cluster-id.scylla.com'] + \
+                    ['{}.cluster-id.scylla.com'.format(host_id) for _, _, host_id in nodes_info]
+        generate_ssl_stores(tmp_dir, dns_names=dns_names)
+        distutils.dir_util.copy_tree(tmp_dir, cluster.get_path())
+
+
+if __name__ == "__main__":
+    from ccmlib.cmds.command import Cmd
+    from ccmlib import common
+
+    a = Cmd()
+    a.path = common.get_default_path()
+    a.cluster = a._load_current_cluster()
+    nodes_info = get_cluster_info(a.cluster)
+    conf_dir = a.cluster.get_path()
+    docker_id, host, port = start_sni_proxy(conf_dir=conf_dir, nodes_info=nodes_info)
+    print(create_cloud_config(conf_dir, host, port))
+    stop_sni_proxy(docker_id)

ccmlib/utils/ssl_utils.py

@@ -0,0 +1,63 @@
+import os
+import subprocess
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def generate_ssl_stores(base_dir, passphrase='cassandra', dns_names=None):
+    """
+    Util for generating ssl stores using java keytool -- nondestructive method if stores already exist this method is
+    a no-op.
+
+    @param base_dir (str) directory where keystore.jks, truststore.jks and ccm_node.cer will be placed
+    @param passphrase (Optional[str]) currently ccm expects a passphrase of 'cassandra' so it's the default but it can be
+            overridden for failure testing
+    @return None
+    @throws CalledProcessError If the keytool fails during any step
+    """
+
+    if os.path.exists(os.path.join(base_dir, 'keystore.jks')):
+        print("keystores already exists - skipping generation of ssl keystores")
+        return
+
+    legacy = ['-legacy'] if '-legacy' in subprocess.run(['openssl', 'pkcs12', '--help'],
+                                                      universal_newlines=True, stderr=subprocess.PIPE).stderr else ''
+    dns_names = dns_names or ['any.cluster-id.scylla.com']
+    ext = ",".join(["dns:{}".format(name) for name in dns_names])
+    print("generating keystore.jks in [{0}]".format(base_dir))
+    subprocess.check_call(['keytool', '-genkeypair', '-alias', 'ccm_node', '-keyalg', 'RSA', '-validity', '365',
+                           '-keystore', os.path.join(base_dir, 'keystore.jks'), '-storepass', passphrase,
+                           '-dname', 'cn=Cassandra Node,ou=CCMnode,o=DataStax,c=US', '-keypass', passphrase,
+                           '-ext', 'san={}'.format(ext)])
+
+    print("exporting cert from keystore.jks in [{0}]".format(base_dir))
+    subprocess.check_call(['keytool', '-export', '-rfc', '-alias', 'ccm_node',
+                           '-keystore', os.path.join(base_dir, 'keystore.jks'),
+                           '-file', os.path.join(base_dir, 'ccm_node.cer'), '-storepass', passphrase])
+    print("importing cert into truststore.jks in [{0}]".format(base_dir))
+    subprocess.check_call(['keytool', '-import', '-file', os.path.join(base_dir, 'ccm_node.cer'),
+                           '-alias', 'ccm_node', '-keystore', os.path.join(base_dir, 'truststore.jks'),
+                           '-storepass', passphrase, '-noprompt'])
+    # Added for scylla: Generate pem format cert/key
+    print("exporting cert to pks12 from keystore.jks in [{0}]".format(base_dir))
+    subprocess.check_call(['keytool', '-importkeystore', '-srckeystore', os.path.join(base_dir, 'keystore.jks'),
+                           '-srcstorepass', passphrase, '-srckeypass', passphrase, '-destkeystore',
+                           os.path.join(base_dir, 'ccm_node.p12'), '-deststoretype', 'PKCS12',
+                           '-srcalias', 'ccm_node', '-deststorepass', passphrase, '-destkeypass', passphrase])
+    print("Using openssl to split pks12 in [{0}] to pem format".format(base_dir))
+    subprocess.check_call(['openssl', 'pkcs12', '-in', os.path.join(base_dir, 'ccm_node.p12'),
+                           '-passin', 'pass:{0}'.format(passphrase), '-nokeys',
+                           '-out', os.path.join(base_dir, 'ccm_node.pem')] + legacy)
+    # Key with password. We want without...
+    subprocess.check_call(['openssl', 'pkcs12', '-in', os.path.join(base_dir, 'ccm_node.p12'),
+                           '-passin', 'pass:{0}'.format(passphrase),
+                           '-passout', 'pass:{0}'.format(passphrase), '-nocerts',
+                           '-out', os.path.join(base_dir, 'ccm_node.tmp')] + legacy)
+    subprocess.check_call(['openssl', 'rsa', '-in', os.path.join(base_dir, 'ccm_node.tmp'),
+                           '-passin', 'pass:{0}'.format(passphrase),
+                           '-out', os.path.join(base_dir, 'ccm_node.key')])
+
+
+if __name__ == "__main__":
+    generate_ssl_stores('/home/fruch/ccm_ssl', dns_names=['any.cluster-id.scylla.com'])