Python RestConnection.add_node方法代码示例

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def _add_node_already_added_body(self):
        self.common_setUp(False)
        master = self.servers[0]
        master_rest = RestConnection(master)
        for i in range(1,len(self.servers)):
            ip = self.servers[i].ip
            self.log.info('adding node : {0} to the cluster'.format(ip))
            otpNode = master_rest.add_node(user=self.membase.rest_username,
                                           password=self.membase.rest_password,
                                           remoteIp=ip, port=self.servers[i].port)
            if otpNode:
                self.log.info('added node : {0} to the cluster'.format(otpNode.id))
                #try to add again
                try:
                    readd_otpNode = master_rest.add_node(user=self.membase.rest_username,
                                           password=self.membase.rest_password,
                                           remoteIp=ip,port=self.servers[i].port)
                    if readd_otpNode:
                        self.fail("server did not raise any exception when calling add_node on an already added node")
                except ServerAlreadyJoinedException:
                    self.log.info("server raised ServerAlreadyJoinedException as expected")

                #now lets eject it
                self.log.info("ejecting the node {0}".format(otpNode.id))
                ejected = master_rest.eject_node(user=self.membase.rest_username,
                                                 password=self.membase.rest_password,
                                                 otpNode=otpNode.id)
                self.assertTrue(ejected,
                                msg="unable to eject the node {0}".format(otpNode.id))
            else:
                self.fail(msg="unable to add node : {0} to the cluster".format(ip))

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def test_add_remove_graceful_add_back_node_with_cert(self,recovery_type=None):
        recovery_type = self.input.param('recovery_type')
        rest = RestConnection(self.master)
        known_nodes = ['[email protected]'+self.master.ip]
        progress = None
        count = 0
        servs_inout = self.servers[1:]
        serv_out = '[email protected]' + servs_inout[1].ip

        rest.create_bucket(bucket='default', ramQuotaMB=100)

        x509main(self.master).setup_master()
        x509main().setup_cluster_nodes_ssl(servs_inout)
        for server in servs_inout:
            rest.add_node('Administrator','password',server.ip)
            known_nodes.append('[email protected]' + server.ip)

        rest.rebalance(known_nodes)
        self.assertTrue(self.check_rebalance_complete(rest),"Issue with rebalance")

        for server in servs_inout:
            status = x509main(server)._validate_ssl_login()
            self.assertEqual(status,200,"Not able to login via SSL code")

        rest.fail_over(serv_out,graceful=True)
        self.assertTrue(self.check_rebalance_complete(rest),"Issue with rebalance")
        rest.set_recovery_type(serv_out,recovery_type)
        rest.add_back_node(serv_out)
        rest.rebalance(known_nodes)
        self.assertTrue(self.check_rebalance_complete(rest),"Issue with rebalance")

        for server in servs_inout:
            status = x509main(server)._validate_ssl_login()
            self.assertEqual(status,200,"Not able to login via SSL code")

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
 def add_nodes(self, task_manager):
     master = self.servers[0]
     rest = RestConnection(master)
     for node in self.to_add:
         self.log.info("adding node {0}:{1} to cluster".format(node.ip, node.port))
         rest.add_node(master.rest_username, master.rest_password,
                       node.ip, node.port)

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
 def test_add_remove_add_back_node_with_cert(self,rebalance=None):
     rebalance = self.input.param('rebalance')
     rest = RestConnection(self.master)
     servs_inout = self.servers[1:3]
     serv_out = '[email protected]' + servs_inout[1].ip
     known_nodes = ['[email protected]'+self.master.ip]
     x509main(self.master).setup_master()
     x509main().setup_cluster_nodes_ssl(servs_inout)
     for server in servs_inout:
         rest.add_node('Administrator','password',server.ip)
         known_nodes.append('[email protected]' + server.ip)
     rest.rebalance(known_nodes)
     self.assertTrue(self.check_rebalance_complete(rest),"Issue with rebalance")
     for server in servs_inout:
         status = x509main(server)._validate_ssl_login()
         self.assertEqual(status,200,"Not able to login via SSL code")
     rest.fail_over(serv_out,graceful=False)
     if (rebalance):
         rest.rebalance(known_nodes,[serv_out])
         self.assertTrue(self.check_rebalance_complete(rest),"Issue with rebalance")
         rest.add_node('Administrator','password',servs_inout[1].ip)
     else:
         rest.add_back_node(serv_out)
     rest.rebalance(known_nodes)
     self.assertTrue(self.check_rebalance_complete(rest),"Issue with rebalance")
     for server in servs_inout:
         response = x509main(server)._validate_ssl_login()
         self.assertEqual(status,200,"Not able to login via SSL code")

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def rebalance_in(self, how_many):
        rest = RestConnection(self._servers[0])
        nodes = rest.node_statuses()
        #choose how_many nodes from self._servers which are not part of
        # nodes
        nodeIps = [node.ip for node in nodes]
        self.log.info("current nodes : {0}".format(nodeIps))
        toBeAdded = []
        selection = self._servers[1:]
        shuffle(selection)
        for server in selection:
            if not server.ip in nodeIps:
                toBeAdded.append(server)
            if len(toBeAdded) == how_many:
                break

        for server in toBeAdded:
            rest.add_node('Administrator', 'password', server.ip)
            #check if its added ?
        nodes = rest.node_statuses()
        otpNodes = [node.id for node in nodes]
        started = rest.rebalance(otpNodes, [])
        msg = "rebalance operation started ? {0}"
        self.log.info(msg.format(started))
        if started:
            result = rest.monitorRebalance()
            msg = "successfully rebalanced out selected nodes from the cluster ? {0}"
            self.log.info(msg.format(result))
            return result
        return False

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def restart_cbrecover_multiple_failover_swapout_reb_routine(self):
        self.common_preSetup()
        when_step = self._input.param("when_step", "recovery_when_rebalance")
        if self._failover is not None:
            if "source" in self._failover:
                rest = RestConnection(self.src_master)
                if self._default_bucket:
                    self.initial_node_count = len(self.src_nodes)
                    self.vbucket_map_before = rest.fetch_vbucket_map()  # JUST FOR DEFAULT BUCKET AS OF NOW
                if self._failover_count >= len(self.src_nodes):
                    raise Exception("Won't failover .. count exceeds available servers on source : SKIPPING TEST")
                if len(self._floating_servers_set) < self._add_count:
                    raise Exception("Not enough spare nodes available, to match the failover count : SKIPPING TEST")
                self.log.info("Failing over {0} nodes on source ..".format(self._failover_count))
                self.failed_nodes = self.src_nodes[(len(self.src_nodes) - self._failover_count):len(self.src_nodes)]
                self.cluster.failover(self.src_nodes, self.failed_nodes)
                for node in self.failed_nodes:
                    self.src_nodes.remove(node)
                add_nodes = self._floating_servers_set[0:self._add_count]
                for node in add_nodes:
                    rest.add_node(user=node.rest_username, password=node.rest_password, remoteIp=node.ip, port=node.port)
                self.src_nodes.extend(add_nodes)
                self.sleep(self.wait_timeout / 4)
                # CALL THE CBRECOVERY ROUTINE WITHOUT WAIT FOR COMPLETED
                self.cbr_routine(self.dest_master, self.src_master, False)

                if "create_bucket_when_recovery" in when_step:
                     name = 'standard_bucket'
                     try:
                         self.cluster.create_standard_bucket(self.src_master, name, STANDARD_BUCKET_PORT + 10, 100, 1)
                     except BucketCreationException, e:
                         self.log.info("bucket creation failed during cbrecovery as expected")
                     # but still able to create bucket on destination
                     self.cluster.create_standard_bucket(self.dest_master, name, STANDARD_BUCKET_PORT + 10, 100, 1)
                     # here we try to re-call cbrecovery(seems it's supported even it's still running)
                     # if recovery fast(=completed) we can get "No recovery needed"
                     self.cbr_routine(self.dest_master, self.src_master)
                elif "recovery_when_rebalance" in when_step:
                    rest.remove_all_recoveries()
                    self.trigger_rebalance(rest, 15)
                    try:
                        self.cbr_routine(self.dest_master, self.src_master)
                        self.log.exception("cbrecovery should be failed when rebalance is in progress")
                    except CBRecoveryFailedException, e:
                        self.log.info("cbrecovery failed  as expected when there are no failovered nodes")
                    reached = RestHelper(rest).rebalance_reached()
                    self.assertTrue(reached, "rebalance failed or did not completed")
                    if self._replication_direction_str == "unidirection":
                        self.log.warn("we expect data lost on source cluster with unidirection replication")
                        self.log.warn("verification data will be skipped")
                        return
                elif "recovery_when_rebalance_stopped" in when_step:
                    rest.remove_all_recoveries()
                    self.trigger_rebalance(rest, 15)
                    rest.stop_rebalance()
                    try:
                        self.cbr_routine(self.dest_master, self.src_master)
                        self.log.exception("cbrecovery should be failed when rebalance has been stopped")
                    except CBRecoveryFailedException, e:
                        self.log.info("cbrecovery failed  as expected when there are no failovered nodes")

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def add_nodes_with_ddoc_ops(self):

        # load initial documents
        self._load_doc_data_all_buckets()

        #create ddocs
        for bucket in self.buckets:
            self._execute_ddoc_ops("create", self.test_with_view, self.num_ddocs, self.num_views_per_ddoc, "dev_test", "v1")

        #execute ddocs asynchronously
        for bucket in self.buckets:
            if self.ddoc_ops == "create":
                #create some more ddocs
                tasks_ddoc = self._async_execute_ddoc_ops(self.ddoc_ops, self.test_with_view, self.num_ddocs / 2, self.num_views_per_ddoc / 2, "dev_test_1", "v1")
            elif self.ddoc_ops in ["update", "delete"]:
                #update delete the same ddocs
                tasks_ddoc = self._async_execute_ddoc_ops(self.ddoc_ops, self.test_with_view, self.num_ddocs / 2, self.num_views_per_ddoc / 2, "dev_test", "v1")

        rest = RestConnection(self.master)
        for node in self.servers[1:]:
            self.log.info("adding node {0}:{1} to cluster".format(node.ip, node.port))
            rest.add_node(self.master.rest_username, self.master.rest_password, node.ip, node.port)

        for task in tasks_ddoc:
            task.result(self.wait_timeout * 2)

        self.verify_cluster_stats(servers=self.servers[:self.nodes_init])
        self._verify_ddoc_ops_all_buckets()
        if self.test_with_view:
            self._verify_ddoc_data_all_buckets()

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def test_add_remove_autofailover(self):
        rest = RestConnection(self.master)
        serv_out = self.servers[3]
        shell = RemoteMachineShellConnection(serv_out)
        known_nodes = ['[email protected]'+self.master.ip]

        rest.create_bucket(bucket='default', ramQuotaMB=100)
        rest.update_autofailover_settings(True,30)

        x509main(self.master).setup_master()
        x509main().setup_cluster_nodes_ssl(self.servers[1:4])
        for server in self.servers[1:4]:
            rest.add_node('Administrator','password',server.ip)
            known_nodes.append('[email protected]'+server.ip)

        rest.rebalance(known_nodes)
        self.assertTrue(self.check_rebalance_complete(rest),"Issue with rebalance")

        shell.stop_server()
        self.sleep(60)
        shell.start_server()
        self.sleep(30)
        for server in self.servers:
            status = x509main(server)._validate_ssl_login()
            self.assertEqual(status,200,"Not able to login via SSL code")

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def test_basic_xdcr_with_cert_regenerate(self):

        cluster1 = self.servers[0:2]
        cluster2 = self.servers[2:4]
        remote_cluster_name = 'sslcluster'
        restCluster1 = RestConnection(cluster1[0])
        restCluster2 = RestConnection(cluster2[0])

        try:
            #Setup cluster1
            x509main(cluster1[0]).setup_master()
            x509main(cluster1[1])._setup_node_certificates(reload_cert=False)

            restCluster1.add_node('Administrator','password',cluster1[1].ip)
            known_nodes = ['[email protected]'+cluster1[0].ip,'[email protected]' + cluster1[1].ip]
            restCluster1.rebalance(known_nodes)
            self.assertTrue(self.check_rebalance_complete(restCluster1),"Issue with rebalance")
            restCluster1.create_bucket(bucket='default', ramQuotaMB=100)
            restCluster1.remove_all_replications()
            restCluster1.remove_all_remote_clusters()

            #Setup cluster2
            x509main(cluster2[0]).setup_master()
            x509main(cluster2[1])._setup_node_certificates(reload_cert=False)

            restCluster2.add_node('Administrator','password',cluster2[1].ip)
            known_nodes = ['[email protected]'+cluster2[0].ip,'[email protected]' + cluster2[1].ip]
            restCluster2.rebalance(known_nodes)
            self.assertTrue(self.check_rebalance_complete(restCluster2),"Issue with rebalance")
            restCluster2.create_bucket(bucket='default', ramQuotaMB=100)

            test = x509main.CACERTFILEPATH + x509main.CACERTFILE
            data  =  open(test, 'rb').read()
            restCluster1.add_remote_cluster(cluster2[0].ip,cluster2[0].port,'Administrator','password',remote_cluster_name,certificate=data)
            replication_id = restCluster1.start_replication('continuous','default',remote_cluster_name)

            #restCluster1.set_xdcr_param('default','default','pauseRequested',True)

            x509main(self.master)._delete_inbox_folder()
            x509main(self.master)._generate_cert(self.servers,root_cn="CB\ Authority")
            self.log.info ("Setting up the first cluster for new certificate")

            x509main(cluster1[0]).setup_master()
            x509main(cluster1[1])._setup_node_certificates(reload_cert=False)
            self.log.info ("Setting up the second cluster for new certificate")
            x509main(cluster2[0]).setup_master()
            x509main(cluster2[1])._setup_node_certificates(reload_cert=False)

            status = restCluster1.is_replication_paused('default','default')
            if not status:
                restCluster1.set_xdcr_param('default','default','pauseRequested',False)

            restCluster1.set_xdcr_param('default','default','pauseRequested',True)
            status = restCluster1.is_replication_paused('default','default')
            self.assertTrue(status,"Replication has not started after certificate upgrade")
        finally:
            known_nodes = ['[email protected]'+cluster2[0].ip,'[email protected]'+cluster2[1].ip]
            restCluster2.rebalance(known_nodes,['[email protected]' + cluster2[1].ip])
            self.assertTrue(self.check_rebalance_complete(restCluster2),"Issue with rebalance")
            restCluster2.delete_bucket()

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
 def _config_cluster(self, nodes):
     master = nodes[0]
     rest = RestConnection(master)
     for node in nodes[1:]:
         rest.add_node(master.rest_username, master.rest_password, node.ip, node.port)
     servers = rest.node_statuses()
     rest.rebalance(otpNodes=[node.id for node in servers], ejectedNodes=[])
     time.sleep(5)

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def cbrecover_multiple_failover_swapout_reb_routine(self):
        self.common_preSetup()
        if self._failover is not None:
            if "source" in self._failover:
                rest = RestConnection(self.src_master)
                if self._default_bucket:
                    self.initial_node_count = len(self.src_nodes)
                    self.vbucket_map_before = rest.fetch_vbucket_map()  # JUST FOR DEFAULT BUCKET AS OF NOW
                if self._failover_count >= len(self.src_nodes):
                    raise Exception("Won't failover .. count exceeds available servers on source : SKIPPING TEST")
                if len(self._floating_servers_set) < self._add_count:
                    raise Exception("Not enough spare nodes available, to match the failover count : SKIPPING TEST")
                self.log.info("Failing over {0} nodes on source ..".format(self._failover_count))
                self.failed_nodes = self.src_nodes[(len(self.src_nodes) - self._failover_count):len(self.src_nodes)]
                self.cluster.failover(self.src_nodes, self.failed_nodes)
                for node in self.failed_nodes:
                    self.src_nodes.remove(node)
                add_nodes = self._floating_servers_set[0:self._add_count]
                for node in add_nodes:
                    rest.add_node(user=node.rest_username, password=node.rest_password, remoteIp=node.ip, port=node.port)
                self.src_nodes.extend(add_nodes)
                self.sleep(self.wait_timeout / 4)
                # CALL THE CBRECOVERY ROUTINE
                self.cbr_routine(self.dest_master, self.src_master)

                self.trigger_rebalance(rest)
                if self._default_bucket:
                    self.vbucket_map_after = rest.fetch_vbucket_map()
                    self.final_node_count = len(self.src_nodes)

            elif "destination" in self._failover:
                rest = RestConnection(self.dest_master)
                if self._default_bucket:
                    self.initial_node_count = len(self.dest_nodes)
                    self.vbucket_map_before = rest.fetch_vbucket_map()  # JUST FOR DEFAULT BUCKET AS OF NOW
                if self._failover_count >= len(self.dest_nodes):
                    raise Exception("Won't failover .. count exceeds available servers on sink : SKIPPING TEST")
                if len(self._floating_servers_set) < self._add_count:
                    raise Exception("Not enough spare nodes available, to match the failover count : SKIPPING TEST")
                self.log.info("Failing over {0} nodes on destination ..".format(self._failover_count))
                self.failed_nodes = self.dest_nodes[(len(self.dest_nodes) - self._failover_count):len(self.dest_nodes)]
                self.cluster.failover(self.dest_nodes, self.failed_nodes)
                for node in self.failed_nodes:
                    self.dest_nodes.remove(node)
                add_nodes = self._floating_servers_set[0:self._add_count]
                for node in add_nodes:
                    rest.add_node(user=node.rest_username, password=node.rest_password, remoteIp=node.ip, port=node.port)
                self.dest_nodes.extend(add_nodes)
                self.sleep(self.wait_timeout / 4)
                # CALL THE CBRECOVERY ROUTINE
                self.cbr_routine(self.src_master, self.dest_master)

                self.trigger_rebalance(rest)
                if self._default_bucket:
                    self.vbucket_map_after = rest.fetch_vbucket_map()
                    self.final_node_count = len(self.dest_nodes)

        self.common_tearDown_verification()

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
 def test_add_node_without_cert(self):
     rest = RestConnection(self.master)
     servs_inout = self.servers[1]
     x509main(self.master).setup_master()
     try:
         rest.add_node('Administrator','password',servs_inout.ip)
     except Exception, ex:
         ex = str(ex)
         expected_result  = "Error adding node: " + servs_inout.ip + " to the cluster:" + self.master.ip + " - [\"Prepare join failed. Error applying node certificate. Unable to read certificate chain file\"]"
         #self.assertEqual(ex,expected_result)
         self.assertTrue(expected_result in ex,"Incorrect Error message in exception")

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
    def _common_test_body(self):
        master = self.servers[0]
        rest = RestConnection(master)
        creds = self.input.membase_settings
        rebalanced_servers = [master]
        bucket_data = RebalanceBaseTest.bucket_data_init(rest)

        self.log.info("INTIAL LOAD")
        RebalanceBaseTest.load_all_buckets_task(rest, self.task_manager, bucket_data, self.load_ratio,
            keys_count=self.keys_count)

        for name in bucket_data:
            for thread in bucket_data[name]["threads"]:
                bucket_data[name]["items_inserted_count"] += thread.inserted_keys_count()

        for server in self.servers[1:]:
            self.log.info("current nodes : {0}".format(RebalanceHelper.getOtpNodeIds(master)))
            #do this 2 times , start rebalance , failover the node , remove the node and rebalance
            for i in range(0, self.num_rebalance):
                distribution = RebalanceBaseTest.get_distribution(self.load_ratio)
                RebalanceBaseTest.load_data_for_buckets(rest, self.load_ratio, distribution, [master], bucket_data,
                    self)
                self.log.info("adding node {0} and rebalance afterwards".format(server.ip))
                otpNode = rest.add_node(creds.rest_username, creds.rest_password, server.ip, server.port)
                msg = "unable to add node {0} to the cluster {1}"
                self.assertTrue(otpNode, msg.format(server.ip, master.ip))
                rest.rebalance(otpNodes=[node.id for node in rest.node_statuses()], ejectedNodes=[])
                self.assertTrue(rest.monitorRebalance(),
                    msg="rebalance operation failed after adding node {0}".format(server.ip))
                rebalanced_servers.append(server)
                RebalanceBaseTest.replication_verification(master, bucket_data, self.replica, self, True)
                rest.fail_over(otpNode.id)
                self.log.info("failed over {0}".format(otpNode.id))
                time.sleep(10)
                rest.rebalance(otpNodes=[node.id for node in rest.node_statuses()],
                    ejectedNodes=[otpNode.id])
                msg = "rebalance failed while removing failover nodes {0}".format(otpNode.id)
                self.assertTrue(rest.monitorRebalance(), msg=msg)
                #now verify the numbers again ?
                RebalanceBaseTest.replication_verification(master, bucket_data, self.replica, self, True)
                #wait 6 minutes
                time.sleep(6 * 60)

            self.log.info("adding node {0} and rebalance afterwards".format(server.ip))
            otpNode = rest.add_node(creds.rest_username, creds.rest_password, server.ip, server.port)
            msg = "unable to add node {0} to the cluster {1}"
            self.assertTrue(otpNode, msg.format(server.ip, master.ip))
            distribution = RebalanceBaseTest.get_distribution(self.load_ratio)
            RebalanceBaseTest.load_data_for_buckets(rest, self.load_ratio, distribution, rebalanced_servers, bucket_data, self)
            rest.rebalance(otpNodes=[node.id for node in rest.node_statuses()], ejectedNodes=[])
            self.assertTrue(rest.monitorRebalance(),
                msg="rebalance operation failed after adding node {0}".format(server.ip))
            rebalanced_servers.append(server)
            RebalanceBaseTest.replication_verification(master, bucket_data, self.replica, self, True)

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
 def rest_api_addNode(self):
     hostnames = self.convert_to_hostname(self, self.servers[0:2])
     master_rest = RestConnection(self.master)
     master_rest.add_node(self.servers[1].rest_username, self.servers[1].rest_password, hostnames[1], self.servers[1].port)
     #Now check whether the node which we added is still referred via hostname or not.
     obj = RestConnection(self.servers[1])
     var = obj.get_nodes_self().hostname
     flag = True if self.servers[1].ip in var else False
     self.assertEqual(flag, False, msg="Fail - Name of node {0} got converted to IP. Failing the test!!!".format(self.servers[1].ip))
     self.log.info("Test Passed!!")
     self.sleep(10)

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import add_node [as 别名]
 def add_remove_servers_and_rebalance(self, to_add, to_remove):
     """
     Add and/or remove servers and rebalance.
     :param to_add: List of nodes to be added.
     :param to_remove: List of nodes to be removed.
     """
     serverinfo = self.servers[0]
     rest = RestConnection(serverinfo)
     for node in to_add:
         rest.add_node(user=serverinfo.rest_username, password=serverinfo.rest_password,
                       remoteIp=node.ip)
     self.shuffle_nodes_between_zones_and_rebalance(to_remove)