Python hashing.ConsistentHashRing类代码示例

 def test_chr_remove_node_missing(self):
     hosts = [("127.0.0.1", "cache0"),("127.0.0.1", "cache1"),("127.0.0.1", "cache2")]
     hashring = ConsistentHashRing(hosts)
     self.assertEqual(hashring.nodes, set(hosts))
     hashring.remove_node(("127.0.0.1", "cache4"))
     self.assertEqual(hashring.nodes, set(hosts))
     self.assertEqual(hashring.nodes_len, 3)

 def test_chr_add_node_duplicate(self):
     hosts = [("127.0.0.1", "cache0"),("127.0.0.1", "cache1"),("127.0.0.1", "cache2")]
     hashring = ConsistentHashRing(hosts)
     self.assertEqual(hashring.nodes, set(hosts))
     hashring.add_node(("127.0.0.1", "cache2"))
     self.assertEqual(hashring.nodes, set(hosts))
     self.assertEqual(hashring.nodes_len, 3)

 def test_chr_get_node_fnv1a(self):
     hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"), ("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
              ("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
     hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
     self.assertEqual(hashring.get_node('hosts.worker1.cpu'),
                      ('127.0.0.1', 'ba603c36342304ed77953f84ac4d357b'))
     self.assertEqual(hashring.get_node('hosts.worker2.cpu'),
                      ('127.0.0.3', '866a18b81f2dc4649517a1df13e26f28'))

 def __init__(self, hosts, timeout):
   self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
   self.ports = dict( ((server, instance), port) for (server, port, instance) in hosts )
   self.timeout = float(timeout)
   self.hash_ring = ConsistentHashRing(self.hosts)
   self.connections = {}
   self.last_failure = {}
   # Create a connection pool for each host
   for host in self.hosts:
     self.connections[host] = set()

  def __init__(self, hosts, timeout):
    self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
    self.ports = dict( ((server, instance), port) for (server, port, instance) in hosts )
    self.timeout = float(timeout)
    servers = set([server for (server, port, instance) in hosts])
    if len(servers) < settings.REPLICATION_FACTOR:
      raise Exception("REPLICATION_FACTOR=%d cannot exceed servers=%d" % (settings.REPLICATION_FACTOR, len(servers)))

    self.hash_ring = ConsistentHashRing(self.hosts)
    self.keyfunc = load_keyfunc()
    self.connections = {}
    self.last_failure = {}
    # Create a connection pool for each host
    for host in self.hosts:
      self.connections[host] = set()

 def test_chr_compute_ring_position_fnv1a(self):
     hosts = [("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"),("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"),
     ("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28")]
     hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch')
     self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'), 59573)
     self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'), 35749)

 def test_chr_get_nodes(self):
     hosts = [("127.0.0.1", "cache0"),("127.0.0.1", "cache1"),("127.0.0.1", "cache2")]
     hashring = ConsistentHashRing(hosts)
     node = hashring.get_nodes('hosts.worker1.cpu')
     self.assertEqual(node, [('127.0.0.1', 'cache2'), ('127.0.0.1', 'cache0'), ('127.0.0.1', 'cache1')])

 def test_chr_compute_ring_position(self):
     hosts = [("127.0.0.1", "cache0"),("127.0.0.1", "cache1"),("127.0.0.1", "cache2")]
     hashring = ConsistentHashRing(hosts)
     self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'), 64833)
     self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'), 38509)

class CarbonLinkPool(object):
  def __init__(self, hosts, timeout):
    self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
    self.ports = dict(
      ((server, instance), port) for (server, port, instance) in hosts )
    self.timeout = float(timeout)
    servers = set([server for (server, port, instance) in hosts])
    if len(servers) < settings.REPLICATION_FACTOR:
      raise Exception("REPLICATION_FACTOR=%d cannot exceed servers=%d" % (
        settings.REPLICATION_FACTOR, len(servers)))

    self.hash_ring = ConsistentHashRing(
      self.hosts, hash_type=settings.CARBONLINK_HASHING_TYPE)
    self.keyfunc = load_keyfunc()
    self.connections = {}
    self.last_failure = {}
    # Create a connection pool for each host
    for host in self.hosts:
      self.connections[host] = set()

  def select_host(self, metric):
    "Returns the carbon host that has data for the given metric"
    key = self.keyfunc(metric)
    nodes = []
    servers = set()
    for node in self.hash_ring.get_nodes(key):
      (server, instance) = node
      if server in servers:
        continue
      servers.add(server)
      nodes.append(node)
      if len(servers) >= settings.REPLICATION_FACTOR:
        break

    available = [ n for n in nodes if self.is_available(n) ]
    return random.choice(available or nodes)

  def is_available(self, host):
    now = time.time()
    last_fail = self.last_failure.get(host, 0)
    return (now - last_fail) < settings.CARBONLINK_RETRY_DELAY

  def get_connection(self, host):
    # First try to take one out of the pool for this host
    (server, instance) = host
    port = self.ports[host]
    connectionPool = self.connections[host]
    try:
      return connectionPool.pop()
    except KeyError:
      pass #nothing left in the pool, gotta make a new connection

    log.cache("CarbonLink creating a new socket for %s" % str(host))
    connection = socket.socket()
    connection.settimeout(self.timeout)
    try:
      connection.connect((server, port))
    except:
      self.last_failure[host] = time.time()
      raise
    else:
      connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
      return connection

  def query(self, metric):
    request = dict(type='cache-query', metric=metric)
    results = self.send_request(request)
    log.cache("CarbonLink cache-query request for %s returned %d datapoints" % (
      metric, len(results['datapoints'])))
    return results['datapoints']

  def get_metadata(self, metric, key):
    request = dict(type='get-metadata', metric=metric, key=key)
    results = self.send_request(request)
    log.cache("CarbonLink get-metadata request received for %s:%s" % (metric, key))
    return results['value']

  def set_metadata(self, metric, key, value):
    request = dict(type='set-metadata', metric=metric, key=key, value=value)
    results = self.send_request(request)
    log.cache("CarbonLink set-metadata request received for %s:%s" % (metric, key))
    return results

  def send_request(self, request):
    metric = request['metric']
    serialized_request = pickle.dumps(request, protocol=-1)
    len_prefix = struct.pack("!L", len(serialized_request))
    request_packet = len_prefix + serialized_request
    result = {}
    result.setdefault('datapoints', [])

    if metric.startswith(settings.CARBON_METRIC_PREFIX):
      return self.send_request_to_all(request)

    if not self.hosts:
      log.cache("CarbonLink is not connected to any host. Returning empty nodes list")
      return result

    host = self.select_host(metric)
    conn = self.get_connection(host)
#.........这里部分代码省略.........