* Issue #32 - multinode start

* Set theme jekyll-theme-minimal * add end of multi cluster * cbbackup exit code check * cbbackupmgr working with sudo, bucket list in snapshot in JSON * check mount points / doc updates * checks added, upgrade added * doc fixes * docs with multi node * fixing uid/gid regex * initial support for using non couchbase user through sudo * issue #22 - initial deployment, index support changes for backup ingestion, various fixes * new logo * partial work * proper style * resync of xdcr and backupmgr OK * v1.1.0 build 1.1.0.2 * v1.1.1 build 1.1.0.2 * ver 1.0.3.1 * ver 1.0.3.2 - sudo for all * version 1.0.2 * version 1.0.3 - error messages fix * version 1.1.0 build 0 * version 1.1.1 - added field for hostname * version 1.1.2 * workinf with copy config * working multinode * working with indexes - tested with backup only - ee, analytics removed from schema * xdcr fixed for multi IP, and different scenarios CE-59 Couchbase Plugin : Conversion of python2 to python3 (#38) Co-authored-by: Marcin Przepiorowski <[email protected]> Co-authored-by: ayesharustgi-delphix <[email protected]> DXE-683 Rename Select Connector - CouchBase DXE-988 Couchbase - VDB failed to enable (#36) Version 1.1.1 (#31) security fix draft
delphix · May 25, 2023 · 73303b9 · 73303b9
1 parent 2229226
commit 73303b9
Show file tree

Hide file tree

Showing 6 changed files with 904 additions and 47 deletions.
diff --git a/src/controller/couchbase_lib/_bucket.py b/src/controller/couchbase_lib/_bucket.py
@@ -39,10 +39,14 @@ def bucket_edit(self, bucket_name, flush_value=1):
         logger.debug("Editing bucket: {} ".format(bucket_name))
         self.__validate_bucket_name(bucket_name)
         env = _BucketMixin.generate_environment_map(self)
-        command = CommandFactory.bucket_edit(bucket_name=bucket_name, flush_value=flush_value, **env)
-        kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
+        kwargs = {ENV_VAR_KEY: {
+            'password': self.parameters.couchbase_admin_password}}
+        env.update(kwargs[ENV_VAR_KEY])
+        # command = CommandFactory.bucket_edit(bucket_name=bucket_name, flush_value=flush_value, **env)
+        command, env_vars = CommandFactory.bucket_edit_expect(bucket_name=bucket_name, flush_value=flush_value, **env)
+        kwargs[ENV_VAR_KEY].update(env_vars)
         logger.debug("edit bucket {}".format(command))
-        return utilities.execute_bash(self.connection, command, **kwargs)
+        return utilities.execute_expect(self.connection, command, **kwargs)
 
     def bucket_edit_ramquota(self, bucket_name, _ramsize):
         """
@@ -53,31 +57,45 @@ def bucket_edit_ramquota(self, bucket_name, _ramsize):
         # It requires the before bucket delete
         logger.debug("Editing bucket: {} ".format(bucket_name))
         self.__validate_bucket_name(bucket_name)
+        kwargs = {ENV_VAR_KEY: {
+            'password': self.parameters.couchbase_admin_password}}
         env = _BucketMixin.generate_environment_map(self)
-        command = CommandFactory.bucket_edit_ramquota(bucket_name=bucket_name, ramsize=_ramsize, **env)
-        kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
+        env.update(kwargs[ENV_VAR_KEY])
+        # command = CommandFactory.bucket_edit_ramquota(bucket_name=bucket_name, ramsize=_ramsize, **env)
+        command, env_vars = CommandFactory.bucket_edit_ramquota_expect(bucket_name=bucket_name,
+                                                      ramsize=_ramsize, **env)
+        kwargs[ENV_VAR_KEY].update(env_vars)
         logger.debug("edit ram bucket {}".format(command))
-        return utilities.execute_bash(self.connection, command, **kwargs)
+        return utilities.execute_expect(self.connection, command, **kwargs)
 
     def bucket_delete(self, bucket_name):
         # To delete the bucket
         logger.debug("Deleting bucket: {} ".format(bucket_name))
         self.__validate_bucket_name(bucket_name)
         env = _BucketMixin.generate_environment_map(self)
-        command = CommandFactory.bucket_delete(bucket_name=bucket_name, **env)
-        kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
+        kwargs = {ENV_VAR_KEY: {
+            'password': self.parameters.couchbase_admin_password}}
+        env.update(kwargs[ENV_VAR_KEY])
+        # command = CommandFactory.bucket_delete(bucket_name=bucket_name, **env)
+        command, env_vars = CommandFactory.bucket_delete_expect(bucket_name=bucket_name, **env)
+        kwargs[ENV_VAR_KEY].update(env_vars)
         logger.debug("delete bucket {}".format(command))
-        return utilities.execute_bash(self.connection, command, **kwargs)
+        return utilities.execute_expect(self.connection, command, **kwargs)
 
     def bucket_flush(self, bucket_name):
         # It requires the before bucket delete
         logger.debug("Flushing bucket: {} ".format(bucket_name))
         self.__validate_bucket_name(bucket_name)
         env = _BucketMixin.generate_environment_map(self)
-        command = CommandFactory.bucket_flush(bucket_name=bucket_name, **env)
-        kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
+        kwargs = {ENV_VAR_KEY: {
+            'password': self.parameters.couchbase_admin_password}}
+        env.update(kwargs[ENV_VAR_KEY])
+        # command, env_vars = CommandFactory.bucket_flush(bucket_name=bucket_name, **env)
+        command, env_vars = CommandFactory.bucket_flush_expect(
+            bucket_name=bucket_name, **env)
+        kwargs[ENV_VAR_KEY].update(env_vars)
         logger.debug("flush bucket {}".format(command))
-        return utilities.execute_bash(self.connection, command, **kwargs)
+        return utilities.execute_expect(self.connection, command, **kwargs)
 
     def bucket_remove(self, bucket_name):
         logger.debug("Removing bucket: {} ".format(bucket_name))
@@ -107,10 +125,14 @@ def bucket_create(self, bucket_name, ram_size, bucket_type, bucket_compression):
 
         policy = self.parameters.bucket_eviction_policy
         env = _BucketMixin.generate_environment_map(self)
-        command = CommandFactory.bucket_create(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, bucket_type=bucket_type, bucket_compression=bucket_compression, **env)
-        kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
+        kwargs = {ENV_VAR_KEY: {
+            'password': self.parameters.couchbase_admin_password}}
+        env.update(kwargs[ENV_VAR_KEY])
+        # command = CommandFactory.bucket_create(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, bucket_type=bucket_type, bucket_compression=bucket_compression, **env)
+        command, env_vars = CommandFactory.bucket_create_expect(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, bucket_type=bucket_type, bucket_compression=bucket_compression, **env)
         logger.debug("create bucket {}".format(command))
-        output, error, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
+        kwargs[ENV_VAR_KEY].update(env_vars)
+        output, error, exit_code = utilities.execute_expect(self.connection, command, **kwargs)
         logger.debug("create bucket output: {} {} {}".format(output, error, exit_code))
         helper_lib.sleepForSecond(2)
 
@@ -120,10 +142,17 @@ def bucket_list(self, return_type=list):
         # It will return also other information like ramused, ramsize etc
         logger.debug("Finding staged bucket list")
         env = _BucketMixin.generate_environment_map(self)
-        command = CommandFactory.bucket_list(**env)
-        kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
+        kwargs = {ENV_VAR_KEY: {
+            'password': self.parameters.couchbase_admin_password}}
+        env.update(kwargs[ENV_VAR_KEY])
+        # command = CommandFactory.bucket_list(**env)
+        command, env_vars = CommandFactory.bucket_list_expect(**env)
+        kwargs[ENV_VAR_KEY].update(env_vars)
         logger.debug("list bucket {}".format(command))
-        bucket_list, error, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
+        # bucket_list, error, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
+        bucket_list, error, exit_code = utilities.execute_expect(self.connection,
+                                                               command,
+                                                               **kwargs)
         logger.debug("list bucket output{}".format(bucket_list))
         if return_type == list:
             #bucket_list = bucket_list.split("\n")
@@ -143,7 +172,6 @@ def bucket_list(self, return_type=list):
         logger.debug("Bucket details in staged environment: {}".format(bucket_list))
         return bucket_list_dict
 
-
     def move_bucket(self, bucket_name, direction):
         logger.debug("Rename folder")
 
@@ -172,11 +200,14 @@ def monitor_bucket(self, bucket_name, staging_UUID):
         # To monitor the replication
         logger.debug("Monitoring the replication for bucket {} ".format(bucket_name))
         kwargs = {ENV_VAR_KEY: {'password': self.staged_source.parameters.xdcr_admin_password}}
-        command = CommandFactory.monitor_replication(source_username=self.staged_source.parameters.xdcr_admin,
+        env = kwargs[ENV_VAR_KEY]
+        command, env_vars = CommandFactory.monitor_replication_expect(source_username=self.staged_source.parameters.xdcr_admin,
                                                      source_hostname=self.source_config.couchbase_src_host,
                                                      source_port=self.source_config.couchbase_src_port,
-                                                     bucket_name=bucket_name, uuid=staging_UUID)
-        stdout, stderr, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
+                                                     bucket_name=bucket_name, uuid=staging_UUID,
+                                                     **env)
+        kwargs[ENV_VAR_KEY].update(env_vars)
+        stdout, stderr, exit_code = utilities.execute_expect(self.connection, command, **kwargs)
         logger.debug("stdout: {}".format(stdout))
         content = json.loads(stdout)
         pending_docs = self._get_last_value_of_node_stats(list(content["nodeStats"].values())[0])

diff --git a/src/controller/couchbase_lib/_cluster.py b/src/controller/couchbase_lib/_cluster.py
@@ -92,8 +92,13 @@ def cluster_setting(self):
         kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
         cluster_name = self._get_cluster_name()
         env = _ClusterMixin.generate_environment_map(self)
-        cmd = CommandFactory.cluster_setting(cluster_name=cluster_name, **env)
-        stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
+        env.update(kwargs[ENV_VAR_KEY])
+        # cmd = CommandFactory.cluster_setting(cluster_name=cluster_name, **env)
+        cmd, env_vars = CommandFactory.cluster_setting_expect(cluster_name=cluster_name, **env)
+        # stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
+        kwargs[ENV_VAR_KEY].update(env_vars)
+        stdout, stderr, exit_code = utilities.execute_expect(self.connection,
+                                                             cmd, **kwargs)
         if re.search(r"ERROR", str(stdout)):
             logger.error("Cluster modification failed, killing the execution")
             raise Exception(stdout)

diff --git a/src/controller/couchbase_lib/_xdcr.py b/src/controller/couchbase_lib/_xdcr.py
@@ -40,12 +40,20 @@ def generate_environment_map(self):
         return env
 
     def xdcr_delete(self, cluster_name):
-        logger.debug("XDCR deletion for cluster_name {} has started ".format(cluster_name))
-        kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password,
-                                'password': self.parameters.couchbase_admin_password}}
+        logger.debug("XDCR deletion for cluster_name {} has started ".format(
+            cluster_name))
+        kwargs = {ENV_VAR_KEY: {
+            'source_password': self.parameters.xdcr_admin_password,
+            'password': self.parameters.couchbase_admin_password}}
         env = _XDCrMixin.generate_environment_map(self)
-        cmd = CommandFactory.xdcr_delete(cluster_name=cluster_name, **env)
-        stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
+        env.update(kwargs[ENV_VAR_KEY])
+        # cmd = CommandFactory.xdcr_delete(cluster_name=cluster_name, **env)
+        cmd, env_vars = CommandFactory.xdcr_delete_expect(
+            cluster_name=cluster_name, **env)
+        kwargs[ENV_VAR_KEY].update(env_vars)
+        # stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
+        stdout, stderr, exit_code = utilities.execute_expect(self.connection,
+                                                             cmd, **kwargs)
         if exit_code != 0:
             logger.error("XDCR Setup deletion failed")
             if stdout:
@@ -61,18 +69,29 @@ def xdcr_setup(self):
         kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password,
                                 'password': self.parameters.couchbase_admin_password}}
         env = _XDCrMixin.generate_environment_map(self)
-        cmd = CommandFactory.xdcr_setup(cluster_name=self.parameters.stg_cluster_name, **env)
-        stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
+        env.update(kwargs[ENV_VAR_KEY])
+        # cmd = CommandFactory.xdcr_setup(cluster_name=self.parameters.stg_cluster_name, **env)
+        cmd, env_vars = CommandFactory.xdcr_setup_expect(cluster_name=self.parameters.stg_cluster_name, **env)
+        kwargs[ENV_VAR_KEY].update(env_vars)
+        stdout, stderr, exit_code = utilities.execute_expect(self.connection, cmd, **kwargs)
         helper_lib.sleepForSecond(3)
 
     def xdcr_replicate(self, src, tgt):
         try:
             logger.debug("Started XDCR replication for bucket {}".format(src))
             kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}}
             env = _XDCrMixin.generate_environment_map(self)
-            cmd = CommandFactory.xdcr_replicate(source_bucket_name=src, target_bucket_name=tgt,
-                                                cluster_name=self.parameters.stg_cluster_name, **env)
-            stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
+            env.update(kwargs[ENV_VAR_KEY])
+            # cmd = CommandFactory.xdcr_replicate(source_bucket_name=src, target_bucket_name=tgt,
+            #                                     cluster_name=self.parameters.stg_cluster_name, **env)
+            cmd, env_vars = CommandFactory.xdcr_replicate_expect(
+                source_bucket_name=src,
+                target_bucket_name=tgt,
+                cluster_name=self.parameters.stg_cluster_name,
+                **env
+            )
+            kwargs[ENV_VAR_KEY].update(env_vars)
+            stdout, stderr, exit_code = utilities.execute_expect(self.connection, cmd, **kwargs)
             if exit_code != 0:
                 logger.debug("XDCR replication create failed")
                 raise Exception(stdout)

diff --git a/src/controller/couchbase_operation.py b/src/controller/couchbase_operation.py
@@ -73,14 +73,14 @@ def uid(self):
     def gid(self):
         return self.__gid
 
-
     def run_couchbase_command(self, couchbase_command, **kwargs):
         logger.debug('run_couchbase_command')
         logger.debug('couchbase_command: {}'.format(couchbase_command))
         if "password" in kwargs:
-            password = kwargs.pop('password')
+            password = kwargs.get('password')
         else:
             password = self.parameters.couchbase_admin_password
+            kwargs["password"] = password
 
         if "username" in kwargs:
             username = kwargs.pop('username')
@@ -99,14 +99,27 @@ def run_couchbase_command(self, couchbase_command, **kwargs):
             env["newpass"] = kwargs.pop('newpass')
 
         if "source_password" in kwargs:
-            env["source_password"] = kwargs.pop('source_password')
+            env["source_password"] = kwargs.get('source_password')
 
         autoparams = [ "shell_path", "install_path", "username", "port", "sudo", "uid", "hostname"]
 
         new_kwargs = {k: v for k, v in kwargs.items() if k not in autoparams}
-
-        method_to_call = getattr(CommandFactory, couchbase_command)
-        command = method_to_call(shell_path=self.repository.cb_shell_path,
+        if couchbase_command not in ["get_server_list",
+                                     "couchbase_server_info",
+                                     "cb_backup_full",
+                                     "build_index",
+                                     "check_index_build",
+                                     "get_source_bucket_list",
+                                     "get_replication_uuid",
+                                     "get_stream_id",
+                                     "delete_replication",
+                                     "node_init",
+                                     "get_indexes_name",
+                                     "rename_cluster",
+                                     "server_add",
+                                     "rebalance"]:
+            method_to_call = getattr(CommandFactory, couchbase_command)
+            command = method_to_call(shell_path=self.repository.cb_shell_path,
                                  install_path=self.repository.cb_install_path,
                                  username=username,
                                  port=self.parameters.couchbase_port,
@@ -115,8 +128,26 @@ def run_couchbase_command(self, couchbase_command, **kwargs):
                                  hostname=hostname,
                                  **new_kwargs)
 
-        logger.debug("couchbase command to run: {}".format(command))
-        stdout, stderr, exit_code = utilities.execute_bash(self.connection, command, environment_vars=env)
+            logger.debug("couchbase command to run: {}".format(command))
+            stdout, stderr, exit_code = utilities.execute_bash(self.connection, command, environment_vars=env)
+        else:
+            couchbase_command = couchbase_command+"_expect"
+            logger.debug('new_couchbase_command: {}'.format(couchbase_command))
+            method_to_call = getattr(CommandFactory, couchbase_command)
+            command, env_vars = method_to_call(shell_path=self.repository.cb_shell_path,
+                                               install_path=self.repository.cb_install_path,
+                                               username=username,
+                                               port=self.parameters.couchbase_port,
+                                               sudo=self.need_sudo,
+                                               uid=self.uid,
+                                               hostname=hostname,
+                                               **new_kwargs
+                                               )
+            env.update(env_vars)
+            logger.debug("couchbase command to run: {}".format(command))
+            stdout, stderr, exit_code = utilities.execute_expect(self.connection,
+                                                               command,
+                                                               environment_vars=env)
         return [stdout, stderr, exit_code]
 
 
@@ -337,7 +368,7 @@ def status(self, provision=False):
 
         except Exception as error:
             # TODO
-            # rewrite it 
+            # rewrite it
             logger.debug("Exception: {}".format(str(error)))
             if re.search("Unable to connect to host at", str(error)):
                 logger.debug("Couchbase service is not running")