Python ClusterOperationHelper.add_all_nodes_or_assert方法代码示例

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
 def setUp(self):
     self.log = logger.Logger.get_logger()
     self.input = TestInputSingleton.input
     self.assertTrue(self.input, msg="input parameters missing...")
     self.servers = self.input.servers
     self.master = self.servers[0]
     rest = RestConnection(self.master)
     rest.init_cluster(username=self.master.rest_username,
                       password=self.master.rest_password)
     info = rest.get_nodes_self()
     node_ram_ratio = BucketOperationHelper.base_bucket_ratio(self.servers)
     rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
     BucketOperationHelper.delete_all_buckets_or_assert(servers=self.servers, test_case=self)
     ClusterOperationHelper.cleanup_cluster(servers=self.servers)
     credentials = self.input.membase_settings
     ClusterOperationHelper.add_all_nodes_or_assert(master=self.master, all_servers=self.servers, rest_settings=credentials, test_case=self)
     rest = RestConnection(self.master)
     nodes = rest.node_statuses()
     otpNodeIds = []
     for node in nodes:
         otpNodeIds.append(node.id)
     rebalanceStarted = rest.rebalance(otpNodeIds, [])
     self.assertTrue(rebalanceStarted,
                     "unable to start rebalance on master node {0}".format(self.master.ip))
     self.log.info('started rebalance operation on master node {0}'.format(self.master.ip))
     rebalanceSucceeded = rest.monitorRebalance()

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
    def common_setup(self, replica):
        self._input = TestInputSingleton.input
        self._servers = self._input.servers
        first = self._servers[0]
        self.log = logger.Logger().get_logger()
        self.log.info(self._input)
        rest = RestConnection(first)
        for server in self._servers:
            RestHelper(RestConnection(server)).is_ns_server_running()

        ClusterOperationHelper.cleanup_cluster(self._servers)
        BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
        ClusterOperationHelper.add_all_nodes_or_assert(self._servers[0], self._servers, self._input.membase_settings, self)
        nodes = rest.node_statuses()
        otpNodeIds = []
        for node in nodes:
            otpNodeIds.append(node.id)
        info = rest.get_nodes_self()
        bucket_ram = info.mcdMemoryReserved * 3 / 4
        rest.create_bucket(bucket="default",
                           ramQuotaMB=int(bucket_ram),
                           replicaNumber=replica,
                           proxyPort=rest.get_nodes_self().moxi)
        msg = "wait_for_memcached fails"
        ready = BucketOperationHelper.wait_for_memcached(first, "default"),
        self.assertTrue(ready, msg)
        rebalanceStarted = rest.rebalance(otpNodeIds, [])
        self.assertTrue(rebalanceStarted,
                        "unable to start rebalance on master node {0}".format(first.ip))
        self.log.info('started rebalance operation on master node {0}'.format(first.ip))
        rebalanceSucceeded = rest.monitorRebalance()
        # without a bucket this seems to fail
        self.assertTrue(rebalanceSucceeded,
                        "rebalance operation for nodes: {0} was not successful".format(otpNodeIds))
        self.awareness = VBucketAwareMemcached(rest, "default")

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
    def setUp(self):
        self._cleanup_nodes = []
        self._failed_nodes = []
        super(FailoverBaseTest, self).setUp()
        self.bidirectional = self.input.param("bidirectional", False)
        self._value_size = self.input.param("value_size", 256)
        self.dgm_run = self.input.param("dgm_run", True)
        credentials = self.input.membase_settings
        self.add_back_flag = False
        self.during_ops = self.input.param("during_ops", None)

        self.log.info(
            "==============  FailoverBaseTest setup was started for test #{0} {1}==============".format(
                self.case_number, self._testMethodName
            )
        )
        try:
            rest = RestConnection(self.master)
            ClusterOperationHelper.add_all_nodes_or_assert(self.master, self.servers, credentials, self)
            nodes = rest.node_statuses()
            rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
            msg = "rebalance failed after adding these nodes {0}".format(nodes)
            self.assertTrue(rest.monitorRebalance(), msg=msg)
        except Exception, e:
            self.cluster.shutdown()
            self.fail(e)

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
 def _setup_cluster(self):
     rest = RestConnection(self.master)
     credentials = self._input.membase_settings
     ClusterOperationHelper.add_all_nodes_or_assert(self.master, self._servers, credentials, self)
     nodes = rest.node_statuses()
     rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
     msg = "rebalance failed after adding these nodes {0}".format(nodes)
     self.assertTrue(rest.monitorRebalance(), msg=msg)

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
    def rebalance_servers_in(servers, rest_settings, testcase):
        log = logger.Logger().get_logger()
        master = servers[0]
        rest = RestConnection(master)
        ClusterOperationHelper.add_all_nodes_or_assert(master, servers, rest_settings, testcase)

        otpNodeIds = []
        for node in rest.node_statuses():
            otpNodeIds.append(node.id)
        rebalanceStarted = rest.rebalance(otpNodeIds, [])
        testcase.assertTrue(rebalanceStarted,
                            "unable to start rebalance on master node {0}".format(master.ip))
        log.info('started rebalance operation on master node {0}'.format(master.ip))
        rebalanceSucceeded = rest.monitorRebalance()
        testcase.assertTrue(rebalanceSucceeded,
                            "rebalance operation for nodes: {0} was not successful".format(otpNodeIds))

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
    def _cluster_setup(self):
        log = logger.Logger.get_logger()

        replicas = self._input.param("replicas", 1)
        keys_count = self._input.param("keys-count", 0)
        num_buckets = self._input.param("num-buckets", 1)

        bucket_name = "default"
        master = self._servers[0]
        credentials = self._input.membase_settings
        rest = RestConnection(master)
        info = rest.get_nodes_self()
        rest.init_cluster(username=master.rest_username,
                          password=master.rest_password)
        rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
        rest.reset_autofailover()
        ClusterOperationHelper.add_all_nodes_or_assert(master, self._servers, credentials, self)
        bucket_ram = info.memoryQuota * 2 / 3

        if num_buckets == 1:
            rest.create_bucket(bucket=bucket_name,
                               ramQuotaMB=bucket_ram,
                               replicaNumber=replicas,
                               proxyPort=info.moxi)
            ready = BucketOperationHelper.wait_for_memcached(master, bucket_name)
            nodes = rest.node_statuses()
            rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
            buckets = rest.get_buckets()
        else:
            created = BucketOperationHelper.create_multiple_buckets(master, replicas, howmany=num_buckets)
            self.assertTrue(created, "unable to create multiple buckets")
            buckets = rest.get_buckets()
            for bucket in buckets:
                ready = BucketOperationHelper.wait_for_memcached(master, bucket.name)
                self.assertTrue(ready, msg="wait_for_memcached failed")
                nodes = rest.node_statuses()
                rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])

#        self.load_data(master, bucket_name, keys_count)

        for bucket in buckets:
            inserted_keys_cnt = self.load_data(master, bucket.name, keys_count)
            log.info('inserted {0} keys'.format(inserted_keys_cnt))

        msg = "rebalance failed after adding these nodes {0}".format(nodes)
        self.assertTrue(rest.monitorRebalance(), msg=msg)
        self.assertTrue(ready, "wait_for_memcached failed")

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
 def add_node_and_rebalance(self, master, servers):
     ClusterOperationHelper.add_all_nodes_or_assert(master, servers, self.input.membase_settings, self)
     rest = RestConnection(master)
     nodes = rest.node_statuses()
     otpNodeIds = []
     for node in nodes:
         otpNodeIds.append(node.id)
     rebalanceStarted = rest.rebalance(otpNodeIds, [])
     self.assertTrue(rebalanceStarted,
                     "unable to start rebalance on master node {0}".format(master.ip))
     self.log.info('started rebalance operation on master node {0}'.format(master.ip))
     rebalanceSucceeded = rest.monitorRebalance()
     self.assertTrue(rebalanceSucceeded,
                     "rebalance operation for nodes: {0} was not successful".format(otpNodeIds))
     self.log.info('rebalance operaton succeeded for nodes: {0}'.format(otpNodeIds))
     #now remove the nodes
     #make sure its rebalanced and node statuses are healthy
     helper = RestHelper(rest)
     self.assertTrue(helper.is_cluster_healthy, "cluster status is not healthy")
     self.assertTrue(helper.is_cluster_rebalanced, "cluster is not balanced")

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
 def setUp(self):
     self._cleanup_nodes = []
     super(FailoverBaseTest, self).setUp()
     self.bidirectional = self.input.param("bidirectional", False)
     self._value_size = self.input.param("value_size", 256)
     self._failed_nodes = []
     self.dgm_run = self.input.param("dgm_run", True)
     self.gen_create = BlobGenerator('loadOne', 'loadOne_', self._value_size, end=self.num_items)
     self.add_back_flag = False
     self.log.info("==============  FailoverBaseTest setup was started for test #{0} {1}=============="\
                   .format(self.case_number, self._testMethodName))
     rest = RestConnection(self.master)
     credentials = self.input.membase_settings
     ClusterOperationHelper.add_all_nodes_or_assert(self.master, self.servers, credentials, self)
     nodes = rest.node_statuses()
     rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
     msg = "rebalance failed after adding these nodes {0}".format(nodes)
     self.assertTrue(rest.monitorRebalance(), msg=msg)
     self.log.info("==============  FailoverBaseTest setup was finished for test #{0} {1} =============="\
                   .format(self.case_number, self._testMethodName))

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
    def _common_test_body(self):
        master = self.servers[0]
        rest = RestConnection(master)
        creds = self.input.membase_settings
        bucket_data = RebalanceBaseTest.bucket_data_init(rest)

        ClusterHelper.add_all_nodes_or_assert(master, self.servers, creds, self)
        rest.rebalance(otpNodes=[node.id for node in rest.node_statuses()], ejectedNodes=[])
        self.assertTrue(rest.monitorRebalance(),
            msg="rebalance operation failed after adding nodes")

        nodes = rest.node_statuses()

        #dont rebalance out the current node
        while len(nodes) > 1:
            #pick a node that is not the master node
            toBeEjectedNode = RebalanceHelper.pick_node(master)
            distribution = RebalanceBaseTest.get_distribution(self.load_ratio)
            RebalanceBaseTest.load_data_for_buckets(rest, self.load_ratio, distribution, [master], bucket_data, self)
            self.log.info("current nodes : {0}".format([node.id for node in rest.node_statuses()]))
            #let's start/step rebalance three times
            self.log.info("removing node {0} and rebalance afterwards".format(toBeEjectedNode.id))
            rest.fail_over(toBeEjectedNode.id)
            self.log.info("failed over {0}".format(toBeEjectedNode.id))
            time.sleep(10)
            rest.rebalance(otpNodes=[node.id for node in rest.node_statuses()],
                ejectedNodes=[toBeEjectedNode.id])
            expected_progress = 30
            reached = RestHelper(rest).rebalance_reached(expected_progress)
            self.assertTrue(reached, "rebalance failed or did not reach {0}%".format(expected_progress))
            stopped = rest.stop_rebalance()
            self.assertTrue(stopped, msg="unable to stop rebalance")
            time.sleep(20)
            RebalanceBaseTest.replication_verification(master, bucket_data, self.replica, self)
            rest.rebalance(otpNodes=[node.id for node in rest.node_statuses()], ejectedNodes=[toBeEjectedNode.id])
            self.assertTrue(rest.monitorRebalance(),
                msg="rebalance operation failed after adding node {0}".format(toBeEjectedNode.id))
            time.sleep(20)

            RebalanceBaseTest.replication_verification(master, bucket_data, self.replica, self)
            nodes = rest.node_statuses()

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
 def _cluster_setup(self):
     bucket_name = "default"
     master = self._servers[0]
     credentials = self._input.membase_settings
     rest = RestConnection(master)
     info = rest.get_nodes_self()
     rest.init_cluster(username=master.rest_username,
                       password=master.rest_password)
     rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
     rest.reset_autofailover()
     ClusterOperationHelper.add_all_nodes_or_assert(master, self._servers, credentials, self)
     bucket_ram = info.memoryQuota * 2 / 3
     rest.create_bucket(bucket=bucket_name,
                        ramQuotaMB=bucket_ram,
                        proxyPort=info.moxi)
     ready = BucketOperationHelper.wait_for_memcached(master, bucket_name)
     nodes = rest.node_statuses()
     rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
     msg = "rebalance failed after adding these nodes {0}".format(nodes)
     self.assertTrue(rest.monitorRebalance(), msg=msg)
     self.assertTrue(ready, "wait_for_memcached failed")

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
    def _install_and_upgrade(self, initial_version='1.6.5.3',
                             create_buckets=False,
                             insert_data=False,
                             start_upgraded_first=True,
                             load_ratio=-1,
                             roll_upgrade=False,
                             upgrade_path=[]):
        node_upgrade_path = []
        node_upgrade_path.extend(upgrade_path)
        #then start them in whatever order you want
        inserted_keys = []
        log = logger.Logger.get_logger()
        if roll_upgrade:
            log.info("performing a rolling upgrade")
        input = TestInputSingleton.input
        rest_settings = input.membase_settings
        servers = input.servers
        save_upgrade_config = False
        is_amazon = False
        if input.test_params.get('amazon',False):
            is_amazon = True
        # install older build on all nodes
        for server in servers:
            remote = RemoteMachineShellConnection(server)
            rest = RestConnection(server)
            info = remote.extract_remote_info()
            older_build = BuildQuery().find_membase_release_build(deliverable_type=info.deliverable_type,
                                                              os_architecture=info.architecture_type,
                                                              build_version=initial_version,
                                                              product='membase-server-enterprise', is_amazon=is_amazon)

            remote.membase_uninstall()
            remote.couchbase_uninstall()
            remote.execute_command('/etc/init.d/membase-server stop')
            remote.download_build(older_build)
            #now let's install ?
            remote.membase_install(older_build)
            RestHelper(rest).is_ns_server_running(testconstants.NS_SERVER_TIMEOUT)
            rest.init_cluster_port(rest_settings.rest_username, rest_settings.rest_password)
            rest.init_cluster_memoryQuota(memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
            remote.disconnect()

        bucket_data = {}
        master = servers[0]
        if create_buckets:
            #let's create buckets
            #wait for the bucket
            #bucket port should also be configurable , pass it as the
            #parameter to this test ? later

            self._create_default_bucket(master)
            inserted_keys = self._load_data(master, load_ratio)
            _create_load_multiple_bucket(self, master, bucket_data, howmany=2)

        # cluster all the nodes together
        ClusterOperationHelper.add_all_nodes_or_assert(master,
                                                       servers,
                                                       rest_settings, self)
        rest = RestConnection(master)
        nodes = rest.node_statuses()
        otpNodeIds = []
        for node in nodes:
            otpNodeIds.append(node.id)
        rebalanceStarted = rest.rebalance(otpNodeIds, [])
        self.assertTrue(rebalanceStarted,
                        "unable to start rebalance on master node {0}".format(master.ip))
        log.info('started rebalance operation on master node {0}'.format(master.ip))
        rebalanceSucceeded = rest.monitorRebalance()
        self.assertTrue(rebalanceSucceeded,
                        "rebalance operation for nodes: {0} was not successful".format(otpNodeIds))

        if initial_version == "1.7.0" or initial_version == "1.7.1":
            self._save_config(rest_settings, master)

        input_version = input.test_params['version']
        node_upgrade_path.append(input_version)
        #if we dont want to do roll_upgrade ?
        log.info("Upgrade path: {0} -> {1}".format(initial_version, node_upgrade_path))
        log.info("List of servers {0}".format(servers))
        if not roll_upgrade:
            for version in node_upgrade_path:
                if version is not initial_version:
                    log.info("Upgrading to version {0}".format(version))
                    self._stop_membase_servers(servers)
                    if re.search('1.8', version):
                        save_upgrade_config = True

                    appropriate_build = _get_build(servers[0], version, is_amazon=is_amazon)
                    self.assertTrue(appropriate_build.url, msg="unable to find build {0}".format(version))
                    for server in servers:
                        remote = RemoteMachineShellConnection(server)
                        remote.download_build(appropriate_build)
                        remote.membase_upgrade(appropriate_build, save_upgrade_config=save_upgrade_config)
                        RestHelper(RestConnection(server)).is_ns_server_running(testconstants.NS_SERVER_TIMEOUT)

                        #verify admin_creds still set
                        pools_info = RestConnection(server).get_pools_info()
                        self.assertTrue(pools_info['implementationVersion'], appropriate_build.product_version)

                        if start_upgraded_first:
#.........这里部分代码省略.........

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
    def common_test_body(self, keys_count, replica, load_ratio, failover_reason):
        log = logger.Logger.get_logger()
        log.info("keys_count : {0}".format(keys_count))
        log.info("replica : {0}".format(replica))
        log.info("load_ratio : {0}".format(load_ratio))
        log.info("failover_reason : {0}".format(failover_reason))
        master = self._servers[0]
        log.info('picking server : {0} as the master'.format(master))
        rest = RestConnection(master)
        info = rest.get_nodes_self()
        rest.init_cluster(username=master.rest_username,
                          password=master.rest_password)
        rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
        bucket_ram = info.memoryQuota * 2 / 3
        bucket = 'default'
        rest.create_bucket(bucket=bucket,
                           ramQuotaMB=bucket_ram,
                           replicaNumber=replica,
                           proxyPort=info.moxi)
        ready = BucketOperationHelper.wait_for_memcached(master, bucket)
        self.assertTrue(ready, "wait_for_memcached_failed")
        credentials = self._input.membase_settings

        ClusterOperationHelper.add_all_nodes_or_assert(master, self._servers, credentials, self)
        nodes = rest.node_statuses()
        rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
        msg = "rebalance failed after adding these nodes {0}".format(nodes)
        self.assertTrue(rest.monitorRebalance(), msg=msg)

        inserted_keys = FailoverBaseTest.load_data(master, bucket, keys_count, load_ratio)
        inserted_count = len(inserted_keys)
        log.info('inserted {0} keys'.format(inserted_count))

        nodes = rest.node_statuses()
        while (len(nodes) - replica) > 1:
            final_replication_state = RestHelper(rest).wait_for_replication(900)
            msg = "replication state after waiting for up to 15 minutes : {0}"
            self.log.info(msg.format(final_replication_state))
            chosen = RebalanceHelper.pick_nodes(master, howmany=replica)
            for node in chosen:
                #let's do op
                if failover_reason == 'stop_server':
                    self.stop_server(node)
                    log.info("10 seconds delay to wait for membase-server to shutdown")
                    #wait for 5 minutes until node is down
                    self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300),
                                    msg="node status is not unhealthy even after waiting for 5 minutes")
                elif failover_reason == "firewall":
                    RemoteUtilHelper.enable_firewall(self._servers, node, bidirectional=self.bidirectional)
                    self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300),
                                    msg="node status is not unhealthy even after waiting for 5 minutes")

                failed_over = rest.fail_over(node.id)
                if not failed_over:
                    self.log.info("unable to failover the node the first time. try again in  60 seconds..")
                    #try again in 60 seconds
                    time.sleep(75)
                    failed_over = rest.fail_over(node.id)
                self.assertTrue(failed_over, "unable to failover node after {0}".format(failover_reason))
                log.info("failed over node : {0}".format(node.id))
            #REMOVEME -
            log.info("10 seconds sleep after failover before invoking rebalance...")
            time.sleep(10)
            rest.rebalance(otpNodes=[node.id for node in nodes],
                           ejectedNodes=[node.id for node in chosen])
            msg = "rebalance failed while removing failover nodes {0}".format(chosen)
            self.assertTrue(rest.monitorRebalance(), msg=msg)
            FailoverBaseTest.replication_verification(master, bucket, replica, inserted_count, self)

            nodes = rest.node_statuses()
        FailoverBaseTest.verify_data(master, inserted_keys, bucket, self)

# 需要导入模块: from membase.helper.cluster_helper import ClusterOperationHelper [as 别名]
# 或者: from membase.helper.cluster_helper.ClusterOperationHelper import add_all_nodes_or_assert [as 别名]
    def common_test_body(self, replica, failover_reason, load_ratio, age, max_nodes):
        log = logger.Logger.get_logger()
        bucket_name = "default"
        log.info("replica : {0}".format(replica))
        log.info("failover_reason : {0}".format(failover_reason))
        log.info("load_ratio : {0}".format(load_ratio))
        log.info("age : {0}".format(age))
        log.info("max_nodes : {0}".format(max_nodes))
        master = self._servers[0]
        log.info('picking server : {0} as the master'.format(master))
        rest = RestConnection(master)
        info = rest.get_nodes_self()
        rest.init_cluster(username=master.rest_username,
                          password=master.rest_password)
        rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
        rest.update_autofailover_settings(True, age, max_nodes)
        rest.reset_autofailover()
        bucket_ram = info.memoryQuota * 2 / 3
        rest.create_bucket(bucket=bucket_name,
                           ramQuotaMB=bucket_ram,
                           replicaNumber=replica,
                           proxyPort=info.moxi)
        ready = BucketOperationHelper.wait_for_memcached(master, bucket_name)
        self.assertTrue(ready, "wait_for_memcached failed")

        credentials = self._input.membase_settings

        log.info("inserting some items in the master before adding any nodes")
        distribution = {512: 0.4, 1 * 1024: 0.59, 5 * 1024: 0.01}
        if load_ratio > 10:
            distribution = {5 * 1024: 0.4, 10 * 1024: 0.5, 20 * 1024: 0.1}

        ClusterOperationHelper.add_all_nodes_or_assert(master, self._servers, credentials, self)
        nodes = rest.node_statuses()
        rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
        msg = "rebalance failed after adding these nodes {0}".format(nodes)
        self.assertTrue(rest.monitorRebalance(), msg=msg)

        inserted_count, rejected_count =\
        MemcachedClientHelper.load_bucket(servers=self._servers,
                                          ram_load_ratio=load_ratio,
                                          value_size_distribution=distribution,
                                          number_of_threads=1)
        log.info('inserted {0} keys'.format(inserted_count))
        nodes = rest.node_statuses()
        # why are we in this while loop?
        while (len(nodes) - replica) >= 1:
            final_replication_state = RestHelper(rest).wait_for_replication(900)
            msg = "replication state after waiting for up to 15 minutes : {0}"
            self.log.info(msg.format(final_replication_state))
            chosen = AutoFailoverBaseTest.choose_nodes(master, nodes, replica)
            for node in chosen:
                #let's do op
                if failover_reason == 'stop_membase':
                    self.stop_membase(node)
                    log.info("10 seconds delay to wait for membase-server to shutdown")
                    #wait for 5 minutes until node is down
                    self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300),
                                    msg="node status is not unhealthy even after waiting for 5 minutes")
                elif failover_reason == "firewall":
                    self.enable_firewall(node)
                    self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300),
                                    msg="node status is not unhealthy even after waiting for 5 minutes")
            # list pre-autofailover stats
            stats = rest.get_bucket_stats()
            self.log.info("pre-autofail - curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count))
            AutoFailoverBaseTest.wait_for_failover_or_assert(master, replica, age, self)

            # manually fail over any unhealthy:active nodes left, max that we should need to manually failover is replica-max_nodes
            manual_failover_count = replica - max_nodes
            for node in chosen:
                self.log.info("checking {0}".format(node.ip))
                if node.status.lower() == "unhealthy" and node.clusterMembership == "active":
                    msg = "node {0} not failed over and we are over out manual failover limit of {1}"
                    self.assertTrue(manual_failover_count > 0, msg.format(node.ip, (replica - max_nodes)))
                    self.log.info("manual failover {0}".format(node.ip))
                    rest.fail_over(node.id)
                    manual_failover_count -= 1

            stats = rest.get_bucket_stats()
            self.log.info("post-autofail - curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count))
            self.assertTrue(stats["curr_items"] == inserted_count, "failover completed but curr_items ({0}) does not match inserted items ({1})".format(stats["curr_items"], inserted_count))

            log.info("10 seconds sleep after autofailover before invoking rebalance...")
            time.sleep(10)
            rest.rebalance(otpNodes=[node.id for node in nodes],
                           ejectedNodes=[node.id for node in chosen])
            msg="rebalance failed while removing failover nodes {0}".format(chosen)
            self.assertTrue(rest.monitorRebalance(), msg=msg)

            nodes = rest.node_statuses()
            if len(nodes) / (1 + replica) >= 1:
                final_replication_state = RestHelper(rest).wait_for_replication(900)
                msg = "replication state after waiting for up to 15 minutes : {0}"
                self.log.info(msg.format(final_replication_state))
                self.assertTrue(RebalanceHelper.wait_till_total_numbers_match(master,bucket_name,600),
                                msg="replication was completed but sum(curr_items) dont match the curr_items_total")

                start_time = time.time()
                stats = rest.get_bucket_stats()
#.........这里部分代码省略.........