Python HashRing.iterate_nodes方法代码示例

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
class MemcacheRing(memcache.Client):
    """Extends python-memcache so it uses consistent hashing to
    distribute the keys.
    """

    def __init__(self, servers, *k, **kw):
        self.hash_ring = HashRing(servers)

        memcache.Client.__init__(self, servers, *k, **kw)

        self.server_mapping = {}
        for server_uri, server_obj in zip(servers, self.servers):
            self.server_mapping[server_uri] = server_obj

    def _get_server(self, key):
        if type(key) == types.TupleType:
            return memcache.Client._get_server(key)

        for i in range(self._SERVER_RETRIES):
            iterator = self.hash_ring.iterate_nodes(key)
            for server_uri in iterator:
                server_obj = self.server_mapping[server_uri]
                if server_obj.connect():
                    return server_obj, key

        return None, None

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
def test_iterate_nodes():
    simple_list = ['1', '2', '3', '4', '5']
    new_ring = HashRing(simple_list)

    nodes = []
    for node in new_ring.iterate_nodes('a'):
        nodes.append(node)

    assert len(nodes) == len(simple_list)
    for elm in simple_list:
        assert elm in nodes

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
    def take_over(self, req):
        """Attached to ``POST /take-over``

        Takes over databases from another server, that presumably has
        gone offline without notice.

        This goes through all of the local databases, and sees if this
        node was either using the bad node as a backup, or is a backup
        for the bad node.  In either case it finds the new node that
        should be either master or handling the bad node, and sends
        the local database to that server.

        Takes a JSON body with keys:

        `other`: a list of all nodes
        `name`: the name of *this* node
        `bad`: the bad node being removed
        `backups`: the number of backups
        """
        self.assert_is_internal(req)
        status = Response(content_type='text/plain')
        data = req.json
        nodes = data['other']
        self_name = data['name']
        bad_node = data['bad']
        assert self_name != bad_node
        backups = data['backups']
        ring = HashRing(nodes)
        for domain, username, bucket in self.storage.all_dbs():
            assert bucket.startswith('/')
            path = '/' + domain + '/' + username + bucket
            iterator = iter(ring.iterate_nodes(path))
            active_nodes = [iterator.next() for i in xrange(backups + 1)]
            replacement_node = iterator.next()
            # Not all the backups should try to restore the database, so instead
            # just the "first" does it
            restore = False
            if active_nodes[0] == bad_node and active_nodes[1:] and active_nodes[1] == self_name:
                status.write('Master node %s for %s removed\n' % (bad_node, path))
                restore = True
            elif bad_node in active_nodes and active_nodes[0] == self_name:
                status.write('Backup node %s for %s removed\n' % (bad_node, path))
                restore = True
            if not restore:
                continue
            db = self.storage.for_user(domain, username, bucket)
            send = Request.blank(replacement_node + urllib.quote(path) + '?paste',
                                 method='POST', body=''.join(db.encode_db()))
            send.environ['cutout.root'] = req.environ.get('cutout.root')
            resp = forward(send)
            assert resp.status_code == 201, str(resp)
            #status.write('  nodes: %r - %r / %r\n' % (active_nodes, bad_node, self_name))
            status.write('  success, added to %s (from %s)\n' % (replacement_node, self_name))
        return status

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
def test_with_objects():
    simple_list = [Server(1), Server(2), Server(3)]

    new_ring = HashRing(simple_list)

    node = new_ring.get_node('BABU')
    assert node in simple_list

    nodes = []
    for node in new_ring.iterate_nodes('aloha'):
        nodes.append(node)

    assert len(nodes) == len(simple_list)
    for elm in simple_list:
        assert elm in nodes

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
    def query_deprecate(self, req):
        """Responds to ``POST /query-deprecate``

        This is used when a new node is added to the system, and all
        existing nodes are asked for what databases should be assigned
        to the new node.  Any such database will be deprecated, and a
        list of those databases is returned.

        Accepts a JSON body with the keys:

        `other`: list of all nodes
        `name`: the name of this node
        `new`: the node being added
        `backups`: the number of backups to keep

        Returns JSON::

            {"deprecated": [deprecated items]}

        Where the deprecated items are::

            {"path": "/domain/user/bucket",
             "domain": "domain",
             "username": "user",
             "bucket": "bucket"
            }
        """
        self.assert_is_internal(req)
        data = req.json
        nodes = data['other']
        self_name = data['name']
        new_node = data['new']
        backups = data['backups']
        ring = HashRing(nodes + [new_node])
        deprecated = []
        for domain, username, bucket in self.storage.all_dbs():
            assert bucket.startswith('/')
            path = '/' + domain + '/' + username + bucket
            iterator = iter(ring.iterate_nodes(path))
            active_nodes = [iterator.next() for i in xrange(backups + 1)]
            deprecated_node = iterator.next()
            if deprecated_node == self_name and new_node in active_nodes:
                deprecated.append(
                    {'path': path, 'domain': domain, 'username': username, 'bucket': bucket})
                db = self.storage.for_user(domain, username, bucket)
                db.deprecate()
        return Response(json={'deprecated': deprecated})

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
    def remove_self(self, req):
        """Responds to ``POST /remove-self``

        This is a request for this node to gracefully remove itself.
        It will attempt to back up its data to the other nodes that
        should take over.

        This takes a request with the JSON data:

        `name`: the name of this node
        `other`: a list of all nodes (including this)
        `backups`: the number of backups to make

        It responds with a text description of what it did.
        """
        self.assert_is_internal(req)
        status = Response(content_type='text/plain')
        self.storage.disable()
        data = req.json
        self_name = data['name']
        status.write('Disabling node %s\n' % self_name)
        ring = HashRing(data['other'])
        for domain, username, bucket in self.storage.all_dbs():
            assert bucket.startswith('/')
            path = '/' + domain + '/' + username + bucket
            db = self.storage.for_user(domain, username, bucket)
            if db.is_deprecated:
                db.clear()
                continue
            iterator = iter(ring.iterate_nodes(path))
            active_nodes = [iterator.next() for i in xrange(data['backups'] + 1)]
            new_node = iterator.next()
            assert self_name in active_nodes, '%r not in %r' % (self_name, active_nodes)
            status.write('Sending %s to node %s\n' % (path, new_node))
            url = urlparse.urljoin(req.application_url, '/' + new_node)
            send = Request.blank(url + urllib.quote(path) + '?paste',
                                 method='POST', body=''.join(db.encode_db()))
            send.environ['cutout.root'] = req.environ.get('cutout.root')
            resp = forward(send)
            assert resp.status_code == 201, str(resp)
            status.write('  success, deleting\n')
            db.clear()
        self.storage.clear()
        return status

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
class ConsistentHashing(object):
    '''
        Usage:
            serverWithWeights = 
            {
                '192.168.0.1:11212': 1,
                '192.168.0.2:11212': 3,
                '192.168.0.3:11212': 2,
            }
            
            ring = ConsistentHashing(serverWithWeights)
            server = ring.getNode("key")
    '''
    def __init__(self, nodeWithWeights):
        super(ConsistentHashing, self).__init__()
        self.__ring = HashRing(nodeWithWeights.keys(), nodeWithWeights)
    
    def getNode(self, key):
        return self.__ring.get_node(key)
    
    def getNodeIterator(self, key, distinct = True):
        return self.__ring.iterate_nodes(key, distinct)

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
import os, sys
import time

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))

from hash_ring import HashRing

init_servers = ['192.168.0.246:11212',
                '192.168.0.247:11212',
                '192.168.0.249:11212',
                '192.168.0.250:11212',
                '192.168.0.251:11212',
                '192.168.0.252:11212',
                '192.168.0.253:11212',
                '192.168.0.255:11212',
                '192.168.0.256:11212',
                '192.168.0.257:11212',
                '192.168.0.258:11212',
                '192.168.0.259:11212']

ring = HashRing(init_servers)
t_start = time.time()
for n in ring.iterate_nodes('test'):
    print n
print 'Time elapsed %s' % (time.time() - t_start)

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import iterate_nodes [as 别名]
class Application(object):
    """Application to route requests to nodes, and backup nodes"""

    def __init__(self, preload=None, preload_dir=None, backups=1):
        self.subnodes = {}
        self.basedir = preload_dir
        nodes = []
        if preload:
            for i in xrange(preload):
                name = 'node-%03i' % i
                dir = os.path.join(preload_dir, name)
                app = sync.Application(dir=dir)
                self.subnodes[name] = app
                nodes.append(name)
        self.ring = HashRing(nodes)
        self.backups = backups

    @wsgify
    def __call__(self, req):
        first = req.path_info_peek()
        if first in self.subnodes:
            req.path_info_pop()
            return self.subnodes[first]
        path = req.path_info
        iterator = iter(self.ring.iterate_nodes(path))
        subnode_url = iterator.next()
        subnode = SubNode(subnode_url)
        if self.backups and req.method == 'POST':
            backup_to = []
            for i in xrange(self.backups):
                backup_to.append(iterator.next())
            req.headers['X-Backup-To'] = ', '.join(backup_to)
        return req.send(subnode)

    def add_node(self, url, create=False, root=None):
        """Adds a new node, with the given url/name"""
        if create:
            dir = os.path.join(self.basedir, url)
            app = sync.Application(dir=dir)
            self.subnodes[url] = app
        node = SubNode(url)
        node.added(self.ring.nodes, backups=self.backups, root=root)
        self.ring = HashRing(self.ring.nodes + [url])

    def remove_node(self, url, root=None, force=False):
        """Removes the given node (according to its url/name)

        If force=True then the node is removed without its cooperation
        """
        node = SubNode(url)
        node.remove(self.ring.nodes, backups=self.backups, force=force, root=root)
        new_nodes = list(self.ring.nodes)
        new_nodes.remove(url)
        self.ring = HashRing(new_nodes)

    def node_list(self, url):
        """Returns a list of the master node and backup nodes for the
        given request URL"""
        iterator = iter(self.ring.iterate_nodes(url))
        nodes = [iterator.next() for i in xrange(self.backups + 1)]
        return nodes