Python hash_ring.HashRing类代码示例

def test_iterate_nodes():
    simple_list = ['1', '2', '3', '4', '5']
    new_ring = HashRing(simple_list)

    nodes = []
    for node in new_ring.iterate_nodes('a'):
        nodes.append(node)

    assert len(nodes) == len(simple_list)
    for elm in simple_list:
        assert elm in nodes

    def take_over(self, req):
        """Attached to ``POST /take-over``

        Takes over databases from another server, that presumably has
        gone offline without notice.

        This goes through all of the local databases, and sees if this
        node was either using the bad node as a backup, or is a backup
        for the bad node.  In either case it finds the new node that
        should be either master or handling the bad node, and sends
        the local database to that server.

        Takes a JSON body with keys:

        `other`: a list of all nodes
        `name`: the name of *this* node
        `bad`: the bad node being removed
        `backups`: the number of backups
        """
        self.assert_is_internal(req)
        status = Response(content_type='text/plain')
        data = req.json
        nodes = data['other']
        self_name = data['name']
        bad_node = data['bad']
        assert self_name != bad_node
        backups = data['backups']
        ring = HashRing(nodes)
        for domain, username, bucket in self.storage.all_dbs():
            assert bucket.startswith('/')
            path = '/' + domain + '/' + username + bucket
            iterator = iter(ring.iterate_nodes(path))
            active_nodes = [iterator.next() for i in xrange(backups + 1)]
            replacement_node = iterator.next()
            # Not all the backups should try to restore the database, so instead
            # just the "first" does it
            restore = False
            if active_nodes[0] == bad_node and active_nodes[1:] and active_nodes[1] == self_name:
                status.write('Master node %s for %s removed\n' % (bad_node, path))
                restore = True
            elif bad_node in active_nodes and active_nodes[0] == self_name:
                status.write('Backup node %s for %s removed\n' % (bad_node, path))
                restore = True
            if not restore:
                continue
            db = self.storage.for_user(domain, username, bucket)
            send = Request.blank(replacement_node + urllib.quote(path) + '?paste',
                                 method='POST', body=''.join(db.encode_db()))
            send.environ['cutout.root'] = req.environ.get('cutout.root')
            resp = forward(send)
            assert resp.status_code == 201, str(resp)
            #status.write('  nodes: %r - %r / %r\n' % (active_nodes, bad_node, self_name))
            status.write('  success, added to %s (from %s)\n' % (replacement_node, self_name))
        return status

def create_sets(servers):
    server_sets = {}
    for s in servers:
        server_sets[s] = set()

    ring = HashRing(servers)
    for word in palindromes:
        node = ring.get_node(word)
        server_sets[node].add(word)

    return server_sets

class LocatorHandler(BaseHandler, Locator.Iface):
    def __init__(self, peer=None, port=9900):
        self.address = socket.gethostbyname(socket.gethostname())
        self.port = port
        self.peer = peer
        self.ring = HashRing()
        try:
            ping(self.location)
            print 'Uh-oh. Our location responded to a ping!'
            raise socket.error(43, 'Address already in use')
        except NodeNotFound:
            pass
    
    @property
    def here(self):
        "Give the canonical string representation"
        return loc2str(self)
    
    @property
    def location(self):
        "Give the canonical Location"
        return Location(address=self.address, port=self.port)
    
    def join(self, location):
        """
        Parameters:
         - location
        """
        self.add(location, [self.location])
        ping_until_return(location)
        items = self.ring.nodes.difference([loc2str(location)])
    
    def remove(self, location, authorities):
        """
        Parameters:
         - location
         - authorities
        """
        key = loc2str(location)
        self.ring.remove(loc2str(location))
        authorities.append(self.location)
        destinations = select_peers(self.ring.nodes.difference(map(loc2str,authorities)))
        for destination in destinations:
            try:
                remote_call('remove', str2loc(destination), location, authorities)
                break
            except NodeNotFound, tx:
                # enter all nodes as authorities to avoid race conditions
                # lazy invalidation
                self.remove(tx.location, map(str2loc, self.ring.nodes))
        print "removed %s:%d" % (location.address, location.port)

 def url(self, name, node=None, **kwargs):
     name = self.path(name)
     if not name:
         return ""
     node_key = node or self.get_key_by_name(name, **kwargs)
     name = name.lstrip("/")
     urls = self.servers[node_key]["URLS"]
     urls = [urls] if isinstance(urls, basestring) else urls
     if len(urls) > 1:
         ring2 = HashRing(urls)
         base_url = ring2.get_node(node_key)
     else:
         base_url = urls[0]
     return base_url + name

def test_with_objects():
    simple_list = [Server(1), Server(2), Server(3)]

    new_ring = HashRing(simple_list)

    node = new_ring.get_node('BABU')
    assert node in simple_list

    nodes = []
    for node in new_ring.iterate_nodes('aloha'):
        nodes.append(node)

    assert len(nodes) == len(simple_list)
    for elm in simple_list:
        assert elm in nodes

    def receive_slave_failure(self):
	sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)         
	host = socket.gethostname()  
	print 'receiving failure'             
	sock.bind((self.host, 10012))
	global backup_slave_nodes
	while True:
		request, addr = sock.recvfrom(1024)
		#request will be just the slave number
		dead_slaves = json.loads(request)
		print 'Recieved dead slaves : '
		print dead_slaves
		for slave in dead_slaves:
			if slave in self.slave_nodes.keys():
				endpoint = self.slave_nodes[slave]
				del self.slave_nodes[slave]
				del self.backup[slave]
				self.memcache_servers.remove(endpoint)
				print 'after deletion'+str(self.slave_nodes)
				#Now add the backup slave nodes
				self.slave_nodes[slave]=backup_slave_nodes[slave]
				self.backup[slave]=backup_slave_nodes[slave]
				self.memcache_servers.append(backup_slave_nodes[slave])
				print 'after addition'+str(self.slave_nodes)
				#change the ring using the new slave
				self.ring = HashRing(self.memcache_servers)

    def __init__(self,portNumber):
        self.portNumber = portNumber
	self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)         
	self.host = socket.gethostname()  
	print self.host             
	self.sock.bind((self.host, self.portNumber)) 
        self.monitors = dict()
        with open("config/monitor.txt") as myfile:
            for line in myfile:
                name, endpoint = line.partition("=")[::2]
		print endpoint
                self.monitors[name] = endpoint
        self.pool = ThreadPool(10) #TODO : configure this
	self.memcache_servers = []
	self.slave_nodes = dict()
	self.backup = dict()
	self.config = {"result":"New","host":self.host,"port":self.portNumber}
	sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	host,port = self.monitors[self.monitors.keys()[0]].split(":")
	sock.sendto(json.dumps(self.config), (host,int(port)))
	slave_config,addr = self.sock.recvfrom(1024)
	sock.close()
	with open("config/slave.txt",'w') as fin:
		for val in slave_config:
			fin.write(str(val))

	with open("config/slave.txt",'r') as myfile:
		for line in myfile:
			name, endpoints = line.partition("=")[::2]
			endpoint1,endpoint2= endpoints.split(',')
			self.memcache_servers.append(endpoint1)
			self.slave_nodes[name]=endpoint1
			self.backup[name] =endpoint2
	self.ring = HashRing(self.memcache_servers)
	print self.backup

 def __init__(self, addr, port):
     RPCServer.__init__(self, addr, port)
     len_up = len(UPLOAD_SERVERS)
     len_repo = len(REPOSITORY_SERVERS)
     if len_up < len_repo or len_up % len_repo != 0:
         show_error(self, 'failed to initialize')
         raise Exception('failed to initialize')
     addr = localhost()
     if addr not in REPOSITORY_SERVERS:
         show_error(self, 'failed to initialize')
         raise Exception('failed to initialize REPOSITORY_SERVERS')
     for i in range(len(REPOSITORY_SERVERS)):
         if  addr == REPOSITORY_SERVERS[i]:
             break
     total = len_up / len_repo 
     self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
     self._print('upload_servers=%s' % str(self._upload_servers))
     if HDFS:
         self._port = HDFS_PORT
         self._client = HDFSClient()
     else:
         self._port = FTP_PORT
         self._client = FTPClient()
         self._server = FTPServer()
     if REPO_DB:
         self._db = Database(addr=REPO_DB)
     else:
         self._db = Database(addr=addr)
     locks = []
     for _ in range(LOCK_MAX):
         locks.append(Lock())
     self._locks = HashRing(locks)
     if DEBUG:
         self._upload_cnt = 0
         self._download_cnt = 0

class MemcacheRing(memcache.Client):
    """Extends python-memcache so it uses consistent hashing to
    distribute the keys.
    """

    def __init__(self, servers, *k, **kw):
        self.hash_ring = HashRing(servers)

        memcache.Client.__init__(self, servers, *k, **kw)

        self.server_mapping = {}
        for server_uri, server_obj in zip(servers, self.servers):
            self.server_mapping[server_uri] = server_obj

    def _get_server(self, key):
        if type(key) == types.TupleType:
            return memcache.Client._get_server(key)

        for i in range(self._SERVER_RETRIES):
            iterator = self.hash_ring.iterate_nodes(key)
            for server_uri in iterator:
                server_obj = self.server_mapping[server_uri]
                if server_obj.connect():
                    return server_obj, key

        return None, None

 def __init__(self, query):
     Thread.__init__(self)
     self._pool = ThreadPool(processes=4)
     self._workers = HashRing(WORKER_SERVERS)
     self._identity = bytes(uuid.uuid4())
     self._query = query
     self._init_sock()

    def query_deprecate(self, req):
        """Responds to ``POST /query-deprecate``

        This is used when a new node is added to the system, and all
        existing nodes are asked for what databases should be assigned
        to the new node.  Any such database will be deprecated, and a
        list of those databases is returned.

        Accepts a JSON body with the keys:

        `other`: list of all nodes
        `name`: the name of this node
        `new`: the node being added
        `backups`: the number of backups to keep

        Returns JSON::

            {"deprecated": [deprecated items]}

        Where the deprecated items are::

            {"path": "/domain/user/bucket",
             "domain": "domain",
             "username": "user",
             "bucket": "bucket"
            }
        """
        self.assert_is_internal(req)
        data = req.json
        nodes = data['other']
        self_name = data['name']
        new_node = data['new']
        backups = data['backups']
        ring = HashRing(nodes + [new_node])
        deprecated = []
        for domain, username, bucket in self.storage.all_dbs():
            assert bucket.startswith('/')
            path = '/' + domain + '/' + username + bucket
            iterator = iter(ring.iterate_nodes(path))
            active_nodes = [iterator.next() for i in xrange(backups + 1)]
            deprecated_node = iterator.next()
            if deprecated_node == self_name and new_node in active_nodes:
                deprecated.append(
                    {'path': path, 'domain': domain, 'username': username, 'bucket': bucket})
                db = self.storage.for_user(domain, username, bucket)
                db.deprecate()
        return Response(json={'deprecated': deprecated})

 def __init__(self, location=None, base_url=None, **kwargs):
     self.clients = {}
     for server in settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"]:
         srv = settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"][server]
         self.clients[server] = Client(domain=srv["DOMAIN"], trackers=srv["TRACKERS"])
     self.servers = settings.DISTRIBUTED_MOGILEFS_CONFIG["SERVERS"]
     self.ring = HashRing(settings.DISTRIBUTED_MOGILEFS_CONFIG["SLOTS"])
     self.kwargs = kwargs

 def getClient(self, key):
     hosts = self.getActiveConnections()
     if hosts != self.hash_ring.nodes:
         self.hash_ring = HashRing(hosts)
     log.msg("Using %i active hosts" % len(hosts))
     if len(hosts) == 0:
         raise NoServerError("No connected servers remaining.")
     return self.hash_ring.get_node(key)

    def __init__(self, servers, *k, **kw):
        self.hash_ring = HashRing(servers)

        memcache.Client.__init__(self, servers, *k, **kw)

        self.server_mapping = {}
        for server_uri, server_obj in zip(servers, self.servers):
            self.server_mapping[server_uri] = server_obj