Python Cluster.add_node方法代码示例

# 需要导入模块: from cluster import Cluster [as 别名]
# 或者: from cluster.Cluster import add_node [as 别名]

#.........这里部分代码省略.........
        old_status = present_node.status
        if new_status == REDIS_STATUS_KO:
            present_node.setKO()
            if old_status == REDIS_STATUS_OK:
                logger.warn("Node (%s) [%s] has DIED!", present_node.role, server)
                if present_node.is_master():
                    logger.warn("Master is down: promoting a new master...")
                    self._promote_new_master(present_node)
                self._update_zk()
        elif new_status == REDIS_STATUS_OK:
            present_node.setOK()
            if old_status == REDIS_STATUS_KO:
                logger.warn("Node (%s) [%s] has RESURRECTED!", present_node.role, server)
                redis_server = self._get_redis_server(host=present_node.host, port=present_node.port)
                master = self.cluster.get_master()
                if master.is_alive():
                    redis_server.slaveof(master.host, master.port)
                else:
                    self._promote_new_master(master)
                self._update_zk()
        else:
            logger.critical("Worker sent an unknown status: [%r]", new_status)

    def _check_all_workers(self):
        ''' check if all worker daemons are still alive,
            if not then re-start.'''
        for worker in self.list_of_workers:
            if not worker.get_process().is_alive():
                logger.error("Process [%s] has died, restarting it...", worker.get_process().name)
                worker.start()

    def _get_redis_server(self, host, port):
        ''' Loookup the redis_class from a map. If not there,
            creates and put in the map.'''
        key = "%s:%d" % (host, port)
        conn = None
        if key in self.redis_servers_map:
            conn = self.redis_servers_map[key]
        else:
            conn = self.redis_class(host, port)
            self.redis_servers_map[key] = conn
        return conn

    def _build_list_of_workers(self):
        ''' Add nodes to cluster, for each host:port received from configuration.'''
        for i, r in enumerate(self.redis_hosts):
            role=ROLE_SLAVE
            host = r.split(':')[0]
            port = int(r.split(':')[1])
            try:
                role = self._get_redis_server(host=host, port=port).info()['role']
                self.cluster.add_node(host, port, role, REDIS_STATUS_OK)
            except ConnectionError:
                logger.error("Node [%r] not available !!", r)
                self.cluster.add_node(host, port, role, REDIS_STATUS_KO)
            finally:
                redis_server = self._get_redis_server(host=host, port=port)
                self.list_of_workers.append(Worker(redis_server, host, port, self.queue, self.sleep_time))

    def _promote_new_master(self, old_master):
        ''' Promotes the first available node as new master,
            if the old master died and if at least one node is still alive.
            All the remaining nodes are set as slaves of the new master.'''
        logger.info("Enter")
        new_master = self.cluster.promote_new_master(old_master)
        if new_master.is_alive():
            redis_master_server = self._get_redis_server(host=new_master.host, port=new_master.port)
            # set this node as the master
            redis_master_server.slaveof()
            # set all the other alive nods as slaves of the new master
            list_of_OK_slaves = self.cluster.filtered_list(roles=(ROLE_SLAVE,), status=(REDIS_STATUS_OK,))
            for slave in list_of_OK_slaves:
                redis_slave_server = self._get_redis_server(host=slave.host, port=slave.port)
                redis_slave_server.slaveof(host=new_master.host, port=new_master.port)
        else:
            logger.critical("*** Can't promote a new master: all nodes are down! ***")
        logger.info("Exit")

    def _update_zk(self):
        ''' Updates ZooKeeper with the new cluster's configuration.'''
        logger.info("Enter")
        (master_as_tuple, list_of_slaves) = self._get_cluster_representation()
        logger.info("Master: [%r]", master_as_tuple)
        slaves_map = {}
        for i, slave in enumerate(list_of_slaves):
            slaves_map[i] = ("%s:%s" % (slave.host, slave.port), slave.status)
        logger.info("Slaves: [%r]", slaves_map)
        logger.warn("Updating ZooKeeper...")
        self._update_zk_properties(master = master_as_tuple, slaves = slaves_map)
        logger.info("Exit")

    def _update_zk_properties(self, master, slaves):
        ''' Calls the update action on the configured zk_path'''
        self.zk_properties.update(master = master, slaves = slaves)

    def __len__(self):
        return len(self.cluster)

    def __str__(self):
        return str(self.cluster)

# 需要导入模块: from cluster import Cluster [as 别名]
# 或者: from cluster.Cluster import add_node [as 别名]
ec2 = EC2(key, access)

events = Events(key, access, cluster.name())
host = Host(cluster.name(), events)
node = host.get_node()
endpoint = host.get_endpoint()
component = os.path.basename(sys.argv[0])
def log(message, logging='info'):
	events.log(node, component, message, logging)

if __name__ == '__main__':
	log('joining the cluster', 'info')

	log('adding the node to the cluster', 'info')
	# now we are ready to be (added to) the cluster
	cluster.add_node(node, endpoint)
	log('creating a Route53 records', 'info')
	r53_zone.create_record(node, endpoint)
	log('setting the tag', 'info')
	ec2.set_tag(node)

	log('getting the master of the node', 'info')
	master = cluster.get_master(node)
	# if we don't have a master, we ARE the master
	if master == None:
		log('setting the main Route53 record for the cluster', 'info')
		r53_zone.update_record(cluster.name(), endpoint)

		# and make sure we 'run' correctly (no-slave, well-monitored)
		log('set the host to run as master', 'info')
		host.set_master()

# 需要导入模块: from cluster import Cluster [as 别名]
# 或者: from cluster.Cluster import add_node [as 别名]
class TestCluster(unittest.TestCase):

    def setUp(self):
        print "### setUp ###"
        self.cluster = Cluster()

    def test_make_key(self):
        print "test_make_key"
        text = "%s:%s" % (HOST, PORT)
        self.assertEqual(self.cluster._make_key(HOST, PORT), text)

    def test_set_role(self):
        self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
        self.cluster._set_role(HOST, PORT, ROLE_MASTER)
        node = self.cluster.get_node(HOST, PORT)
        self.assertTrue(node.is_master)

    def test_filtered_list(self):
        print "test_filtered_list"
        self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
        self.cluster.add_node(host=HOST, port=PORT+1, role=ROLE_SLAVE, status=REDIS_STATUS_KO)
        self.cluster.add_node(host=HOST, port=PORT+2, role=ROLE_SLAVE, status=REDIS_STATUS_OK)
        testlist = self.cluster.filtered_list(roles=(ROLE_MASTER))
        self.assertEqual(len(testlist), 1)
        self.assertTrue(testlist[0].is_master())
        self.assertTrue(testlist[0].is_alive())
        testlist = self.cluster.filtered_list(roles=(ROLE_SLAVE))
        self.assertEqual(len(testlist), 2)
        self.assertTrue(testlist[0].is_slave() and testlist[1].is_slave())
        testlist = self.cluster.filtered_list()
        self.assertEqual(len(testlist), 3)
        testlist = self.cluster.filtered_list(status=REDIS_STATUS_KO)
        self.assertEqual(len(testlist), 1)
        self.assertTrue(testlist[0].is_slave() and not testlist[0].is_alive())
        testlist = self.cluster.filtered_list(status=REDIS_STATUS_OK)
        self.assertEqual(len(testlist), 2)
        testlist = self.cluster.filtered_list(roles=ROLE_MASTER, status=REDIS_STATUS_KO)
        self.assertEqual(len(testlist), 0)
        testlist = self.cluster.filtered_list(roles=(ROLE_MASTER,ROLE_SLAVE), status=REDIS_STATUS_KO)
        self.assertEqual(len(testlist), 1)
        self.assertTrue(testlist[0].is_slave() and not testlist[0].is_alive())

    def test_add_single_master(self):
        print "test_add_single_master"
        self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
        self.assertEqual(len(self.cluster), 1)
        node = self.cluster.get_master()
        self.assertTrue(node.is_master() and node.is_alive())

    def test_add_single_slave(self):
        print "test_add_single_slave"
        self.cluster.add_node(host=HOST, port=PORT, role=ROLE_SLAVE, status=REDIS_STATUS_OK)
        self.assertEqual(len(self.cluster), 1)
        self.assertIsNone(self.cluster.get_master())
        node = self.cluster.get_node(host=HOST, port=PORT)
        self.assertTrue(node.is_slave() and node.is_alive())

    def test_promote_new_master_single_node(self):
        print "test_promote_new_master_single_node"
        old_master = self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
        self.assertEqual(len(self.cluster), 1)
        self.cluster.promote_new_master(old_master)
        node = self.cluster.get_master()
        self.assertTrue(node is not None)
        self.assertTrue(node.is_master())
        self.assertFalse(node.is_alive())

    def test_str(self):
        print "test_str"
        self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
        self.cluster.add_node(host=HOST, port=PORT+1, role=ROLE_SLAVE, status=REDIS_STATUS_OK)
        self.assertEqual("(localhost:1000,master,OK),(localhost:1001,slave,OK)", str(self.cluster))

    def _create_message(self, redis_host, redis_port, redis_status):
        worker_name = '%s:%s' % (redis_host, redis_port)
        text = "%s,%s" % (worker_name, redis_status)
        return text

    def tearDown(self):
        print "### tearDown ###"
        self.cluster = None

# 需要导入模块: from cluster import Cluster [as 别名]
# 或者: from cluster.Cluster import add_node [as 别名]
def create_clusters(node_data):
    for node_name, node_tags in node_data:
        c =  Cluster(ID = int(node_name))
        c.add_node(node_name, node_tags)
        clusterglobals.clusters_list.append(c)