Python HashRing.get_node方法代码示例

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
class ShardClient(object):

    """
    ::code-block
        servers = [
            {'name': 'server1', 'host': '192.168.0.246', 'port': 6379, 'db': 0, 'weight': 1},
            {'name': 'server2', 'host': '192.168.0.247', 'port': 6379, 'db': 0, 'weight': 1},
            {'name': 'server3', 'host': '192.168.0.248', 'port': 6379, 'db': 0, 'weight': 1},
            {'name': 'server4', 'host': '192.168.0.249', 'port': 6379, 'db': 0, 'weight': 1},
        ]
    """

    def __init__(self, servers):
        self._connections = {}
        hosts, weights = self.format_servers(servers)
        self._ring = HashRing(hosts, weights)
        self.build_connections(servers)

    def format_servers(self, servers):
        hosts = []
        weights = {}
        for s in servers:
            name = self.node_name(s)
            hosts.append(name)
            weights[name] = s['weight']

        return hosts, weights

    def node_name(self, s):
        return ('%s:%s:%s:%s')%(s['name'], s['host'], s['port'], s['db'])

    def build_connections(self, servers):
        for s in servers:
            self._connections[self.node_name(s)] = self.connect_redis(**s)

    def connect_redis(self, host='localhost', port=6379, db=0, **kwargs):
        return redis.StrictRedis(host=host, port=port, db=db)

    def get_server(self, key):
        return self._ring.get_node(key)

    def get_connection(self, key):
        node = self._ring.get_node(key)
        return self._connections[node]

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
class MultiserverClient(Client): 
	"""Client that connects to multiple servers"""
	def __init__(self, servers=servers):
		self.servers = servers
		self.ring = HashRing(servers)

	#################### PRIVATE FUNCTIONS ####################
	def base_url(self, key):
		server = self.ring.get_node(key)
		# print "Using server ", server, " for key ", key
		return 'http://%s' % (server)

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
def create_sets(servers):
    server_sets = {}
    for s in servers:
        server_sets[s] = set()

    ring = HashRing(servers)
    for word in palindromes:
        node = ring.get_node(word)
        server_sets[node].add(word)

    return server_sets

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
 def url(self, name, node=None, **kwargs):
     name = self.path(name)
     if not name:
         return ""
     node_key = node or self.get_key_by_name(name, **kwargs)
     name = name.lstrip("/")
     urls = self.servers[node_key]["URLS"]
     urls = [urls] if isinstance(urls, basestring) else urls
     if len(urls) > 1:
         ring2 = HashRing(urls)
         base_url = ring2.get_node(node_key)
     else:
         base_url = urls[0]
     return base_url + name

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
def test_with_objects():
    simple_list = [Server(1), Server(2), Server(3)]

    new_ring = HashRing(simple_list)

    node = new_ring.get_node('BABU')
    assert node in simple_list

    nodes = []
    for node in new_ring.iterate_nodes('aloha'):
        nodes.append(node)

    assert len(nodes) == len(simple_list)
    for elm in simple_list:
        assert elm in nodes

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
class HashRingGrouper(BaseGrouper):
    """
    Group messages based on a consistent hashing algorithm. e.g. a message with the same grouping key will always
    go to the same intersection
    """
    def __init__(self, *args, **kwargs):
        super(HashRingGrouper, self).__init__(*args, **kwargs)
        self.hash_ring = HashRing(self.destinations)

    def get_destinations_for(self, value):
        if type(value) is int:
            value = str(value)
        try:
            return [self.hash_ring.get_node(value)]
        except TypeError:
            raise GroupingValueMissing("'%s' is an invalid grouping value for HashRingGrouper" % (value, ))

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
class DistRedis(MethodMissing):
    def __init__(self,hosts):
        self.hosts=[]
        for h in hosts:
            host,port = h.split(':')
            self.hosts.append(Redis(host,int(port)))
        self.ring = HashRing(self.hosts)

    def add_server(self,server):
        server,port = server.split(':')
        r= Redis(server,port)
        self.ring.add_node(r)

    def save(self):
        for redis in self.ring:
            redis.save()
    
    def bgsave(self):
        for redis in self.ring:
            redis.save(True)

    def delete_cloud(self):
        for redis in self.ring:
            for key in self.ring.keys("*"):
                redis.delete(key)
    
    def quit(self):
        for redis in self.ring:
            redis.quit

    def node_for_key(self,key):
        if re.match("/\{(.*)\?\}/",key):
            l=re.split("/\{(.*)\?\}/",key)
            key = l[0]
        return self.ring.get_node(key)

    def method_missing(self, attr, *args, **kwargs):
        redis = self.node_for_key(args[0])
        if redis != None:
            return redis.__getattribute__(attr)(*args,**kwargs)


    '''def node_for_key(self,key):

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
	def split_keys(self,keylocation):
		keys_1= {}
		keys_2= {}
		api=MugenDBAPI('temp_dbfile','temp_keyfile')
		for_new = MugenDBAPI('MugenDBfile_{}.txt'.format(self.newnode[0]),'KeyMap_{}.txt'.format(self.newnode[0]))
		servers = [self.slavenum+":10000",self.newnode[1]+":10000"]
		ring = HashRing(servers)
		for key in keylocation:
			with open("MugenDBfile.txt",'r') as myfile:
				myfile.seek(keylocation[key][1],0)
				data = json.loads(myfile.readline())
			hxmd5=calculatemd5(key)
			server=ring.get_node(hxmd5)
			if server == servers[1]:
				for_new.put(key,data,keys_2,keylocation[key][0])
			else:
				api.put(key,data,keys_1,keylocation[key][0])

		os.system('mv temp_dbfile.txt MugenDBfile.txt ')
		os.system('mv temp_keyfile.txt KeyMap.txt ')

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
class ConsistentHashing(object):
    '''
        Usage:
            serverWithWeights = 
            {
                '192.168.0.1:11212': 1,
                '192.168.0.2:11212': 3,
                '192.168.0.3:11212': 2,
            }
            
            ring = ConsistentHashing(serverWithWeights)
            server = ring.getNode("key")
    '''
    def __init__(self, nodeWithWeights):
        super(ConsistentHashing, self).__init__()
        self.__ring = HashRing(nodeWithWeights.keys(), nodeWithWeights)
    
    def getNode(self, key):
        return self.__ring.get_node(key)
    
    def getNodeIterator(self, key, distinct = True):
        return self.__ring.iterate_nodes(key, distinct)

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
def _get_bridge(key):
    ring = HashRing(BRIDGE_SERVERS)
    return '%s:%d' % (ring.get_node(key), BRIDGE_PORT)

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
class MasterMonitorConnection:
    ''' Receive the requests from monitor node and redirect them to slave nodes using consistent hashing'''
    def __init__(self,portNumber):
        self.portNumber = portNumber
	self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)         
	self.host = socket.gethostname()  
	print self.host             
	self.sock.bind((self.host, self.portNumber)) 
        self.monitors = dict()
        with open("config/monitor.txt") as myfile:
            for line in myfile:
                name, endpoint = line.partition("=")[::2]
		print endpoint
                self.monitors[name] = endpoint
        self.pool = ThreadPool(10) #TODO : configure this
	self.memcache_servers = []
	self.slave_nodes = dict()
	self.backup = dict()
	self.config = {"result":"New","host":self.host,"port":self.portNumber}
	sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	host,port = self.monitors[self.monitors.keys()[0]].split(":")
	sock.sendto(json.dumps(self.config), (host,int(port)))
	slave_config,addr = self.sock.recvfrom(1024)
	sock.close()
	with open("config/slave.txt",'w') as fin:
		for val in slave_config:
			fin.write(str(val))

	with open("config/slave.txt",'r') as myfile:
		for line in myfile:
			name, endpoints = line.partition("=")[::2]
			endpoint1,endpoint2= endpoints.split(',')
			self.memcache_servers.append(endpoint1)
			self.slave_nodes[name]=endpoint1
			self.backup[name] =endpoint2
	self.ring = HashRing(self.memcache_servers)
	print self.backup

    def find_slave_node(self,hxmd5):
	'''fetch the slave node to which this req should be redirected'''
	server = self.ring.get_node(hxmd5)
	return server

    def calculatemd5(self,key):
	'''calculate md5 hash for the given key'''
	m = hashlib.md5()
	m.update(key)
	return m.digest()

    def listen(self):
	''' listen and redirect the requests to slaves '''
	print 'listening....' 
	thread = Thread(target = self.receive_slave_failure, args = ())
	thread.start()
	while True:
			  try:
			   	  request, addr = self.sock.recvfrom(1024)			  
				  rec_req = json.loads(request)
				  rec_req['id'] = 1
				  logger.debug('Processing {0} request from userid= {1},data={2}'.format(rec_req['id'],rec_req['userid'],rec_req['data']))		
				  try:
					#rec_req['data'].keys():
					key = rec_req['data'].keys()[0]
				  except:
					key = rec_req['data']
				  print rec_req
				  hxmd5 = self.calculatemd5(key)		  
				  slave_node = self.find_slave_node(hxmd5)
				  print slave_node
				  #rec_req['md5'] = "".join(ord(x) for x in hxmd5)
                                  rec_req['md5'] = "hello"
				  rec_req['Master'] = 'Master'+str(masternum)
				  print rec_req
			  except:
				print (traceback.format_exc())
			  
                          #fetch slave node host and port
			  host,port = slave_node.partition(":")[::2]
			  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
		    	  print host,port
		    	  sock.sendto(json.dumps(rec_req), (host,int(port)))
		          logger.debug('Processed {0} request, sent to {1}'.format(rec_req['id'],slave_node))
		    	  sock.close()
			  if rec_req['request'] == 'get':
			  	self.send_to_neighbours(slave_node,rec_req)

    def send_to_neighbours(self,slave_node,rec_req):
	'''If request is get then we need to fetch the result from 2 neighbouring nodes of the selected slave node.'''
	for name,endpoint in self.slave_nodes.items():
		if endpoint == slave_node:
			slave = name
	slave_num=int(slave[5:])
	count = len(self.memcache_servers)
	#send it to two neighbours
	slave2 = ((slave_num%count)+1)
	slave3 = ((slave_num%count)+2)
        if slave2 > count:
		slave2 = slave2%count
	if slave3 > count:
		slave3 = slave3%count
#.........这里部分代码省略.........

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
from hash_ring import HashRing

weights = {
    '0.0.0.1': 1,
    '0.0.0.2': 2,
    '0.0.0.3': 3,
    '0.0.0.4': 4,
    '0.0.0.5': 5

}

ring = HashRing(weights.keys(), weights)
for i in xrange(0,100000):
    print i, ring.get_node(str(i))

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
def get_bridge(key):
    ring = HashRing(BRIDGE_SERVERS)
    return ring.get_node(key)

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
class Scheduler():
    def __init__(self,):
        '''
        init redis with the config. start a inter thread.
        hashring with replication
        inworker is a thread can push or get worker from it.
        '''
        self.rs=redis.StrictRedis(host=config.redis_host, port=config.redis_port,db=0)
        self.inter=inputer.Inputer(config.input_home)
        self.db=Filedb(config.db_file)
        self.workers=self.db.get()
        self.time=time.time()
        #TODO:add the weight for hashring() init

        self.hashring=HashRing(nodes=self.workers,replicas=100)
        self.nodes={}
    
    def reload_worker(self,):
        ctime=time.time()
        if ctime-self.time>1:
            new_workers=self.db.get()
            if new_workers!=self.workers:
                news=[var for var in new_workers if var not in self.workers]
                for new in news:
                    self.hashring.add_node(new)
                deles=[var for var in self.workers if var not in new_workers]
                for dele in deles:
                    self.hashring.remove_node(dele)
                self.workers=new_workers
            self.time=ctime
        
        
    def pushurl(self,url):
        #TODO:rs push failed!
        node=self.hashring.get_node(url)
        self.rs.lpush(node,url)
        if node not in self.nodes.keys():
            self.nodes[node]=0
        self.nodes[node]+=1
        return node

    def get_nodes(self,):
        for node in self.nodes.keys():
            print node+":"+str(self.nodes[node])

    def run(self,):
        self.inter.start()
        while self.inter.running:
            ifile = self.inter.get_task()
            if ifile == None:
		time.sleep(3)
		continue
            if not os.path.exists(ifile):
		time.sleep(3)
		continue

            try:
		for line in open(ifile):
                    self.reload_worker()
                    if line.strip()!="":
                        node=self.pushurl(line.strip())
                    if self.inter.running == 0:
                        break

                self.get_nodes()
            except Exception,e:
                #TODO:write to log
                print str(e)
		continue

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import get_node [as 别名]
class MQTTProxy(object):
    broker_servers = []
    broker_number = 0
    redis_pool = None
    redis_key_expire = 60*60*24#redis中信息老化时间暂时没有使用
    '''
    broker_servers = ['192.168.1.241:1883',
                        '192.168.1.241:1884',
                        '192.168.1.249:1885']
    '''
    def __init__(self,cfg):
        logging.info('init MQTTProxy.....')
        logging.info("broker servers : " + str(cfg['mqtt_broker_servers']))

        MQTTProxy.redispool = redis.ConnectionPool(host=cfg["mqtt_redis_host"], port=cfg["mqtt_redis_port"],
                                                db=cfg["mqtt_redis_db"])
        MQTTProxy.redis_key_expire = cfg['mqtt_redis_expire']
        MQTTProxy.broker_servers = cfg['mqtt_broker_servers']

        if not isinstance(MQTTProxy.broker_servers,list) or MQTTProxy.broker_servers == []:
            raise RuntimeError ('init MQTTProxy fail....')

        MQTTProxy.broker_number = len(MQTTProxy.broker_servers)
        #如果启动时要检查mqtt broker服务有没开启，可以将这行注释打开
        #self.check_broker_server(self.broker_servers)
        self.ring = HashRing(self.broker_servers)

    def __get_redis_conn(self):
        return redis.StrictRedis(connection_pool=MQTTProxy.redispool)       
  
    def __get_ip_port(self,broker):
        return broker.split(":")

    #根据key获取所对应的broker的ip和port,key合法性在函数外校验
    def get_broker_server_info(self,key):
        redisconn = self.__get_redis_conn()
        broker = redisconn.get(key)
        if broker == None:
            broker = self.dispatch_broker_server(key)
        return broker

    #根据key删除redis中保存的broker的ip和port
    def del_broker_server_info(self,key):
        redisconn = self.__get_redis_conn()
        broker = redisconn.get(key)
        pipe = redisconn.pipeline()
        pipe.multi()
        pipe.delete(key)
        pipe.hdel(broker,key)
        pipe.execute()

    #根据key给用户分配一个broker服务的ip和port
    def dispatch_broker_server(self,key):
        redisconn = self.__get_redis_conn()
        brokerN = self.ring.get_node(key) 
        brokerO = redisconn.get(key)
        if brokerN != brokerO:
           redisconn.hdel(brokerO,key) 
        pipe = redisconn.pipeline()
        pipe.multi()
        pipe.set(key,brokerN)
        pipe.hset(brokerN,key,1)
        pipe.execute()
        return self.__get_ip_port(brokerN)

    #检查所有的broker服务是否已经启动
    def check_broker_server(self,brokerlist):
        try:
            if not isinstance(brokerlist,list):
                brokerlist = list(brokerlist)
            for broker in brokerlist:
                ip,port = self.__get_ip_port(broker)
                client = mqtt.Client()
                client.connect(ip,port)
        except:
            raise RuntimeError("Checking out MQTT broker:broker server is not Running!")

    #获取这个代理管理的所有broker服务器ip和port
    def get_broker_server_list(self):
        return MQTTProxy.broker_servers

    #获取这些key所对应的broker的ip和port
    def get_broker_server_by_keys(self,keys):
        if not isinstance(keys,list):
            raise TypeError("get_broker_server_by keys type is error!")
        redisconn = self.__get_redis_conn()
        brokerlist = redisconn.mget(keys)
        brokerlist=list(set(brokerlist))
        if None in brokerlist:
            brokerlist.remove(None)
        return brokerlist

    #给一个话题topic发送一条消息或者多条消息 
    def sendmessage(self,topic,msg,key):
        if key == None:
            raise TypeError("sendmessage: key is None!")
        broker = self.get_broker_server_info(key)
        ip,port = self.__get_ip_port(broker)
        if isinstance(msg,str):
            MQTT.publish_one(ip,port,topic,msg)
#.........这里部分代码省略.........