Python RestConnection.get_nodes_versions方法代码示例

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
    def add_built_in_server_user(self, testuser=None, rolelist=None, node=None):
        """
           From spock, couchbase server is built with some users that handles
           some specific task such as:
               cbadminbucket
           Default added user is cbadminbucket with admin role
        """
        rest = RestConnection(self.master)
        versions = rest.get_nodes_versions()
        for version in versions:
            if "5" > version:
                self.log.info("Atleast one of the nodes in the cluster is "
                              "pre 5.0 version. Hence not creating rbac user "
                              "for the cluster. RBAC is a 5.0 feature.")
                return
        if testuser is None:
            testuser = [{'id': 'cbadminbucket', 'name': 'cbadminbucket',
                                                'password': 'password'}]
        if rolelist is None:
            rolelist = [{'id': 'cbadminbucket', 'name': 'cbadminbucket',
                                                      'roles': 'admin'}]
        if node is None:
            node = self.master

        self.log.info("**** add built-in '%s' user to node %s ****" % (testuser[0]["name"],
                                                                       node.ip))
        RbacBase().create_user_source(testuser, 'builtin', node)
        self.sleep(10)

        self.log.info("**** add '%s' role to '%s' user ****" % (rolelist[0]["roles"],
                                                                testuser[0]["name"]))
        status = RbacBase().add_user_role(rolelist, RestConnection(node), 'builtin')
        self.sleep(10)
        return status

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
 def create_primary_index_for_3_0_and_greater(self):
     self.log.info("CREATE PRIMARY INDEX using %s" % self.primary_indx_type)
     rest = RestConnection(self.master)
     versions = rest.get_nodes_versions()
     if versions[0].startswith("4") or versions[0].startswith("3") or versions[0].startswith("5"):
         for bucket in self.buckets:
             if self.primary_indx_drop:
                 self.log.info("Dropping primary index for %s using %s ..." % (bucket.name,self.primary_indx_type))
                 self.query = "DROP PRIMARY INDEX ON %s USING %s" % (bucket.name,self.primary_indx_type)
                 #self.run_cbq_query()
                 self.sleep(3, 'Sleep for some time after index drop')
             self.query = 'select * from system:indexes where name="#primary" and keyspace_id = "%s"' % bucket.name
             res = self.run_cbq_query()
             self.sleep(10)
             if self.monitoring:
                 self.query = "delete from system:completed_requests"
                 self.run_cbq_query()
             if not self.skip_primary_index:
                 if (res['metrics']['resultCount'] == 0):
                     self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
                     self.log.info("Creating primary index for %s ..." % bucket.name)
                     try:
                         self.run_cbq_query()
                         self.primary_index_created = True
                         if self.primary_indx_type.lower() == 'gsi':
                             self._wait_for_index_online(bucket, '#primary')
                     except Exception, ex:
                         self.log.info(str(ex))

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
 def create_primary_index_for_3_0_and_greater(self):
     self.log.info("CHECK FOR PRIMARY INDEXES")
     rest = RestConnection(self.master)
     versions = rest.get_nodes_versions()
     ddoc_name = 'ddl_#primary'
     if versions[0].startswith("3"):
         try:
             rest.get_ddoc(self.buckets[0], ddoc_name)
         except ReadDocumentException:
             for bucket in self.buckets:
                 self.log.info("Creating primary index for %s ..." % bucket.name)
                 self.query = "CREATE PRIMARY INDEX ON %s " % (bucket.name)
                 try:
                     self.run_cbq_query()
                 except Exception, ex:
                     self.log.error('ERROR during index creation %s' % str(ex))

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
 def create_primary_index_for_3_0_and_greater(self):
     self.log.info("CREATE PRIMARY INDEX")
     rest = RestConnection(self.master)
     versions = rest.get_nodes_versions()
     if versions[0].startswith("4") or versions[0].startswith("3"):
         for bucket in self.buckets:
             if self.primary_indx_drop:
                 self.log.info("Dropping primary index for %s ..." % bucket.name)
                 self.query = "DROP PRIMARY INDEX ON %s" % (bucket.name)
                 self.sleep(3, 'Sleep for some time after index drop')
             self.log.info("Creating primary index for %s ..." % bucket.name)
             self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
             try:
                 self.run_cbq_query()
                 if self.primary_indx_type.lower() == 'gsi':
                     self._wait_for_index_online(bucket, '#primary')
             except Exception, ex:
                 self.log.info(str(ex))

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
 def create_primary_index_for_3_0_and_greater(self):
     self.log.info("CREATE PRIMARY INDEX using %s" % self.primary_indx_type)
     rest = RestConnection(self.master)
     versions = rest.get_nodes_versions()
     if versions[0].startswith("4") or versions[0].startswith("3"):
         for bucket in self.buckets:
             if self.primary_indx_drop:
                 self.log.info("Dropping primary index for %s using %s ..." % (bucket.name, self.primary_indx_type))
                 self.query = "DROP PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
                 # self.run_cbq_query()
                 self.sleep(3, "Sleep for some time after index drop")
             self.query = 'select * from system:indexes where name="#primary" and keyspace_id = "%s"' % bucket.name
             res = self.run_cbq_query()
             self.sleep(10)
             print res
             if self.monitoring:
                 self.query = "delete from system:completed_requests"
                 self.run_cbq_query()
             if not self.skip_primary_index:
                 if res["metrics"]["resultCount"] == 0:
                     self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
                     self.log.info("Creating primary index for %s ..." % bucket.name)
                     # if self.gsi_type:
                     #     self.query += " WITH {'index_type': 'memdb'}"
                     try:
                         self.run_cbq_query()
                         self.primary_index_created = True
                         if self.primary_indx_type.lower() == "gsi":
                             self._wait_for_index_online(bucket, "#primary")
                     except Exception, ex:
                         self.log.info(str(ex))
             if self.monitoring:
                 self.query = "select * from system:active_requests"
                 result = self.run_cbq_query()
                 print result
                 self.assertTrue(result["metrics"]["resultCount"] == 1)
                 self.query = "select * from system:completed_requests"
                 time.sleep(20)
                 result = self.run_cbq_query()
                 print result

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
 def create_primary_index_for_3_0_and_greater(self):
     self.log.info("CREATE PRIMARY INDEX")
     rest = RestConnection(self.master)
     versions = rest.get_nodes_versions()
     if versions[0].startswith("4") or versions[0].startswith("3"):
         for bucket in self.buckets:
             if self.primary_indx_drop:
                 self.log.info("Dropping primary index for %s ..." % bucket.name)
                 self.query = "DROP PRIMARY INDEX ON %s" % (bucket.name)
                 self.sleep(3, 'Sleep for some time after index drop')
             self.query = "select * from system:indexes where name='#primary' and keyspace_id = %s" % bucket.name
             res = self.run_cbq_query(self.query)
             if (res['metrics']['resultCount'] == 0):
                 self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
                 self.log.info("Creating primary index for %s ..." % bucket.name)
                 if self.gsi_type:
                     self.query += " WITH {'index_type': 'memdb'}"
                 try:
                     self.run_cbq_query()
                     self.primary_index_created= True
                     if self.primary_indx_type.lower() == 'gsi':
                         self._wait_for_index_online(bucket, '#primary')
                 except Exception, ex:
                     self.log.info(str(ex))

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
class FailoverTests(FailoverBaseTest):
    def setUp(self):
        super(FailoverTests, self).setUp(self)

    def tearDown(self):
        super(FailoverTests, self).tearDown(self)

    def test_failover_firewall(self):
        self.common_test_body('firewall')

    def test_failover_normal(self):
        self.common_test_body('normal')

    def test_failover_stop_server(self):
        self.common_test_body('stop_server')

    def test_failover_then_add_back(self):
        self.add_back_flag = True
        self.common_test_body('normal')

    def common_test_body(self, failover_reason):
        """
            Main Test body which contains the flow of the failover basic steps
            1. Starts Operations if programmed into the test case (before/after)
            2. Start View and Index Building operations
            3. Failover K out of N nodes (failover can be HARDFAILOVER/GRACEFUL)
            4.1 Rebalance the cluster is failover of K nodeStatuses
            4.2 Run Add-Back operation with recoveryType = (full/delta) with rebalance
            5. Verify all expected operations completed by checking stats, replicaiton, views, data correctness
        """
        # Pick the reference node for communication
        # We pick a node in the cluster which will NOT be failed over
        self.filter_list = []
        if self.failoverMaster:
            self.master = self.servers[1]
        self.log.info(" Picking node {0} as reference node for test case".format(self.master.ip))
        self.print_test_params(failover_reason)
        self.rest = RestConnection(self.master)
        self.nodes = self.rest.node_statuses()
        # Set the data path for the cluster
        self.data_path = self.rest.get_data_path()

        # Check if the test case has to be run for 3.0.0
        versions = self.rest.get_nodes_versions()
        self.version_greater_than_2_5 = True
        for version in versions:
            if "3" > version:
                self.version_greater_than_2_5 = False

        # Do not run this this test if graceful category is being used
        if not self.version_greater_than_2_5 and (self.graceful or (self.recoveryType != None)):
            self.log.error("Graceful failover can't be applied to nodes with version less then 3.*")
            self.log.error("Please check configuration parameters: SKIPPING TEST.")
            return

        # Find nodes that will under go failover
        if self.failoverMaster:
            self.chosen = RebalanceHelper.pick_nodes(self.master, howmany=1, target_node = self.servers[0])
        else:
            self.chosen = RebalanceHelper.pick_nodes(self.master, howmany=self.num_failed_nodes)

        # Perform operations - Create/Update/Delete
        # self.withMutationOps = True => Run Operations in parallel to failover
        # self.withMutationOps = False => Run Operations Before failover
        self.load_initial_data()
        if not self.withMutationOps:
            self.run_mutation_operations()
        # Perform View Creation Tasks and check for completion if required before failover
        if self.withViewsOps:
            self.run_view_creation_operations(self.servers)
            if not self.createIndexesDuringFailover:
                self.query_and_monitor_view_tasks(self.servers)

        # Take snap-shot of data set used for validaiton
        record_static_data_set ={}
        prev_vbucket_stats = {}
        prev_failover_stats = {}
        if not self.withMutationOps:
            record_static_data_set = self.get_data_set_all(self.servers, self.buckets, path = None)

        # Capture  vbucket and failover stats if test version >= 2.5.*
        if self.version_greater_than_2_5 and self.upr_check:
            prev_vbucket_stats = self.get_vbucket_seqnos(self.servers, self.buckets)
            prev_failover_stats = self.get_failovers_logs(self.servers, self.buckets)

        # Perform Operations relalted to failover
        if self.withMutationOps or self.withViewsOps or self.compact:
            self.run_failover_operations_with_ops(self.chosen, failover_reason)
        else:
            self.run_failover_operations(self.chosen, failover_reason)

        # Perform Add Back Operation with Rebalance Or only Rebalance with Verificaitons
        if not self.gracefulFailoverFail and self.runRebalanceAfterFailover:
            if self.add_back_flag:
                self.run_add_back_operation_and_verify(self.chosen, prev_vbucket_stats, record_static_data_set, prev_failover_stats)
            else:
                self.run_rebalance_after_failover_and_verify(self.chosen, prev_vbucket_stats, record_static_data_set, prev_failover_stats)
        else:
            return

#.........这里部分代码省略.........

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
class FailoverTests(FailoverBaseTest):
    def setUp(self):
        super(FailoverTests, self).setUp(self)

    def tearDown(self):
        super(FailoverTests, self).tearDown(self)

    def test_failover_firewall(self):
        self.common_test_body('firewall')

    def test_failover_normal(self):
        self.common_test_body('normal')

    def test_failover_stop_server(self):
        self.common_test_body('stop_server')

    def test_failover_then_add_back(self):
        self.add_back_flag = True
        self.common_test_body('normal')

    def common_test_body(self, failover_reason):
        """
            Main Test body which contains the flow of the failover basic steps
            1. Starts Operations if programmed into the test case (before/after)
            2. Start View and Index Building operations
            3. Failover K out of N nodes (failover can be HARDFAILOVER/GRACEFUL)
            4.1 Rebalance the cluster is failover of K nodeStatuses
            4.2 Run Add-Back operation with recoveryType = (full/delta) with rebalance
            5. Verify all expected operations completed by checking stats, replicaiton, views, data correctness
        """
        # Pick the reference node for communication
        # We pick a node in the cluster which will NOT be failed over
        self.referenceNode = self.master
        if self.failoverMaster:
            self.referenceNode = self.servers[1]
        self.log.info(" Picking node {0} as reference node for test case".format(self.referenceNode.ip))
        self.print_test_params(failover_reason)
        self.rest = RestConnection(self.referenceNode)
        self.nodes = self.rest.node_statuses()

        # Set the data path for the cluster
        self.data_path = self.rest.get_data_path()

        # Check if the test case has to be run for 3.0.0
        versions = self.rest.get_nodes_versions()
        self.version_greater_than_2_5 = True
        for version in versions:
            if "3" > version:
                self.version_greater_than_2_5 = False

        # Do not run this this test if graceful category is being used
        if not self.version_greater_than_2_5 and (self.graceful or (self.recoveryType != None)):
            self.log.error("Graceful failover can't be applied to nodes with version less then 3.*")
            self.log.error("Please check configuration parameters: SKIPPING TEST.")
            return

        # Find nodes that will under go failover
        self.chosen = RebalanceHelper.pick_nodes(self.referenceNode, howmany=self.num_failed_nodes)

        # Perform operations - Create/Update/Delete
        # self.withOps = True => Run Operations in parallel to failover
        # self.withOps = False => Run Operations Before failover
        self.ops_tasks = self.run_operation_tasks()

        # Perform View Creation Tasks and check for completion if required before failover
        if self.runViews:
            self.run_view_creation_operations(self.servers)
            if not self.runViewsDuringFailover:
                self.run_view_creation_operations(self.servers)
                self.monitor_view_tasks(self.servers)

        # Take snap-shot of data set used for validaiton
        record_static_data_set = self.get_data_set_all(self.servers, self.buckets, path = None)
        prev_vbucket_stats = {}
        prev_failover_stats = {}

        # Capture  vbucket and failover stats if test version >= 2.5.*
        if self.version_greater_than_2_5 and self.upr_check:
            prev_vbucket_stats = self.get_vbucket_seqnos(self.servers, self.buckets)
            prev_failover_stats = self.get_failovers_logs(self.servers, self.buckets)

        # Perform Operations relalted to failover
        self.run_failover_operations(self.chosen, failover_reason)

        # Perform Add Back Operation with Rebalance Or only Rebalance with Verificaitons
        if not self.gracefulFailoverFail:
            if self.add_back_flag:
                self.run_add_back_operation_and_verify(self.chosen, prev_vbucket_stats, record_static_data_set, prev_failover_stats)
            else:
                self.run_rebalance_after_failover_and_verify(self.chosen, prev_vbucket_stats, record_static_data_set, prev_failover_stats)

    def run_rebalance_after_failover_and_verify(self, chosen, prev_vbucket_stats, record_static_data_set, prev_failover_stats):
        """ Method to run rebalance after failover and verify """
        # Need a delay > min because MB-7168
        self.sleep(60, "after failover before invoking rebalance...")
        _servers_ = self.filter_servers(self.servers, chosen)
        # Rebalance after Failover operation
        self.rest.rebalance(otpNodes=[node.id for node in self.nodes],
                               ejectedNodes=[node.id for node in chosen])
        if self.during_ops:
#.........这里部分代码省略.........

# 需要导入模块: from membase.api.rest_client import RestConnection [as 别名]
# 或者: from membase.api.rest_client.RestConnection import get_nodes_versions [as 别名]
    def common_test_body(self, keys_count, failover_reason):
        log = logger.Logger.get_logger()
        log.info("keys_count : {0}".format(keys_count))
        log.info("replicas : {0}".format(self.num_replicas))
        log.info("failover_reason : {0}".format(failover_reason))
        log.info('picking server : {0} as the master'.format(self.master))

        rest = RestConnection(self.master)
        versions = rest.get_nodes_versions()
        for version in versions:
            if "3" > version and self.graceful:
                log.error("Graceful failover can't be applied to nodes with version less then 3.*")
                log.error("Please check configuration parameters: SKIPPING TEST.")
                return

        self._load_all_buckets(self.master, self.gen_create, "create", 0,
                               batch_size=10000, pause_secs=5, timeout_secs=180)
        self._wait_for_stats_all_buckets(self.servers)

        _servers_ = copy.deepcopy(self.servers)
        nodes = rest.node_statuses()

        RebalanceHelper.wait_for_replication(self.servers, self.cluster)
        chosen = RebalanceHelper.pick_nodes(self.master, howmany=self.num_replicas)
        for node in chosen:
            # let's do op
            if failover_reason == 'stop_server':
                self.stop_server(node)
                log.info("10 seconds delay to wait for membase-server to shutdown")
                # wait for 5 minutes until node is down
                self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300),
                                    msg="node status is not unhealthy even after waiting for 5 minutes")
            elif failover_reason == "firewall":
                server = [srv for srv in self.servers if node.ip == srv.ip][0]
                RemoteUtilHelper.enable_firewall(server, bidirectional=self.bidirectional)
                status = RestHelper(rest).wait_for_node_status(node, "unhealthy", 300)
                if status:
                    log.info("node {0}:{1} is 'unhealthy' as expected".format(node.ip, node.port))
                else:
                    # verify iptables on the node if something wrong
                    for server in self.servers:
                        if server.ip == node.ip:
                            shell = RemoteMachineShellConnection(server)
                            info = shell.extract_remote_info()
                            if info.type.lower() == "windows":
                                o, r = shell.execute_command("netsh advfirewall show allprofiles")
                            else:
                                o, r = shell.execute_command("/sbin/iptables --list")
                            shell.log_command_output(o, r)
                            shell.disconnect()
                    rest.print_UI_logs()
                    api = rest.baseUrl + 'nodeStatuses'
                    status, content, header = rest._http_request(api)
                    json_parsed = json.loads(content)
                    self.log.info("nodeStatuses: {0}".format(json_parsed))
                    self.fail("node status is not unhealthy even after waiting for 5 minutes")

            failed_over = rest.fail_over(node.id, graceful=self.graceful)
            if self.graceful and (failover_reason in ['stop_server', 'firewall']):
                if failed_over:
                    # MB-10479
                    rest.print_UI_logs()
                self.assertFalse(failed_over, "Graceful Falover was started for unhealthy node!!! ")
                #perform general Failover
                failed_over = rest.fail_over(node.id)
            if not failed_over:
                self.log.info("unable to failover the node the first time. try again in  60 seconds..")
                # try again in 75 seconds
                self.sleep(75)
                failed_over = rest.fail_over(node.id, graceful=self.graceful)
            if self.graceful and (failover_reason not in ['stop_server', 'firewall']):
                reached = RestHelper(rest).rebalance_reached()
                self.assertTrue(reached, "rebalance failed for Graceful Failover, stuck or did not completed")

            self.assertTrue(failed_over, "unable to failover node after {0}".format(failover_reason))
            log.info("failed over node : {0}".format(node.id))
            self._failed_nodes.append(node)

        if self.add_back_flag:
            for node in self._failed_nodes:
                rest.add_back_node(node.id)
                self.sleep(5)
            self.sleep(10, "after failover before invoking rebalance...")
            rest.rebalance(otpNodes=[node.id for node in nodes],
                               ejectedNodes=[])
            msg = "rebalance failed while removing failover nodes {0}".format(chosen)
            self.assertTrue(rest.monitorRebalance(stop_if_loop=True), msg=msg)
        else:
            # Need a delay > min because MB-7168
            self.sleep(60, "after failover before invoking rebalance...")
            rest.rebalance(otpNodes=[node.id for node in nodes],
                               ejectedNodes=[node.id for node in chosen])
            if self.during_ops:
                self.sleep(5, "Wait for some progress in rebalance")
                if self.during_ops == "change_password":
                    old_pass = self.master.rest_password
                    self.change_password(new_password=self.input.param("new_password", "new_pass"))
                    rest = RestConnection(self.master)
                elif self.during_ops == "change_port":
                    self.change_port(new_port=self.input.param("new_port", "9090"))
#.........这里部分代码省略.........