Python HashRing.add_node方法代码示例

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import add_node [as 别名]
class DistRedis(MethodMissing):
    def __init__(self,hosts):
        self.hosts=[]
        for h in hosts:
            host,port = h.split(':')
            self.hosts.append(Redis(host,int(port)))
        self.ring = HashRing(self.hosts)

    def add_server(self,server):
        server,port = server.split(':')
        r= Redis(server,port)
        self.ring.add_node(r)

    def save(self):
        for redis in self.ring:
            redis.save()
    
    def bgsave(self):
        for redis in self.ring:
            redis.save(True)

    def delete_cloud(self):
        for redis in self.ring:
            for key in self.ring.keys("*"):
                redis.delete(key)
    
    def quit(self):
        for redis in self.ring:
            redis.quit

    def node_for_key(self,key):
        if re.match("/\{(.*)\?\}/",key):
            l=re.split("/\{(.*)\?\}/",key)
            key = l[0]
        return self.ring.get_node(key)

    def method_missing(self, attr, *args, **kwargs):
        redis = self.node_for_key(args[0])
        if redis != None:
            return redis.__getattribute__(attr)(*args,**kwargs)


    '''def node_for_key(self,key):

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import add_node [as 别名]
class NodeIndexer:
    """
    Recebe conexões de supernós e auxilia na estruturação de uma rede overlay
    """

    def __init__(self, port):
        self.connList = []
        self.port = port
        # socket vai usar tcp
        self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # reusar o mesmo socket para outras conexões
        self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        # classe que vai fazer consistent hashing dos supernós
        self.ring = HashRing()

    def serve_forever(self):
        # configura a escuta no localhost, porta especificada
        self.server_socket.bind(('', self.port))
        self.server_socket.listen(10)

        self.connList.append(self.server_socket)

        print "Iniciando o servico na porta " + str(self.port)

        try:
            while 1:
                # pega a lista de sockets que podem ser lidos
                read_sockets,write_sockets,error_sockets = select.select(self.connList,[],[])

                for sock in read_sockets:
                    # nova conexão
                    if sock == self.server_socket:
                        sockfd, addr = self.server_socket.accept()
                        self.connList.append(sockfd)
                        print "Novo superno conectado (%s:%s)" % addr
                    else:
                        # mensagem de um supernó
                        try:
                            data = sock.recv(BUFFER_SIZE)
                            if len(data) != 0:
                                self.handle_data(data, sock)
                        except(KeyboardInterrupt):
                            # para que a interrupcao do teclado nao seja tratada no próximo except
                            raise
                        except(ValueError):
                            print "Mensagem fora das especificações recebida de (%s:%s)" % addr
                            print data

            # se saiu do loop, a conexão foi fechada por alguma razão
            self.server_socket.close()
        except(KeyboardInterrupt):
            # interrupção do teclado
            print "Fechando servico"
            sys.exit(0)

    def handle_data(self, data, socket):
        opr = data[0]
        if opr == 'a':
            # append: informa a porta que o supernó está recebendo conexões
            port = int(data[1:])
            self.handle_add(socket, port)
        elif opr == 'r':
            # remove: supernó avisa que vai desconectar
            port = int(data[1:])
            self.handle_remove(socket, port)
        elif opr == 'd':
            # supernó foi desconectado
            self.handle_disconnect(socket)
        elif opr == 'q':
            # query: requisição de um supernó pelo supernó responsável pelo arquivo descrito por 'data'
            if self.ring.is_empty():
                response = 'empty'
            else:
                response = "%s:%s" % self.ring.get_node(data)
            socket.send(response)
        else:
            # mensagem não interpretada pelo serviço de indexação
            raise ValueError

    def handle_add(self, socket, port):
        """ adiciona o supernó no anel hash """
        addr = (socket.getpeername()[0], port)
        self.ring.add_node(addr)

    def handle_remove(self, socket, port):
        """ retira o supernó da indexação """
        addr = (socket.getpeername()[0], port)
        print "Superno desconectando: (%s, %s)" % socket.getpeername()
        self.ring.remove_node(addr)

    def handle_disconnect(self, socket):
        """ remove a conexão para o supernó """
        socket.close()
        self.connList.remove(socket)

# 需要导入模块: from hash_ring import HashRing [as 别名]
# 或者: from hash_ring.HashRing import add_node [as 别名]
class Scheduler():
    def __init__(self,):
        '''
        init redis with the config. start a inter thread.
        hashring with replication
        inworker is a thread can push or get worker from it.
        '''
        self.rs=redis.StrictRedis(host=config.redis_host, port=config.redis_port,db=0)
        self.inter=inputer.Inputer(config.input_home)
        self.db=Filedb(config.db_file)
        self.workers=self.db.get()
        self.time=time.time()
        #TODO:add the weight for hashring() init

        self.hashring=HashRing(nodes=self.workers,replicas=100)
        self.nodes={}
    
    def reload_worker(self,):
        ctime=time.time()
        if ctime-self.time>1:
            new_workers=self.db.get()
            if new_workers!=self.workers:
                news=[var for var in new_workers if var not in self.workers]
                for new in news:
                    self.hashring.add_node(new)
                deles=[var for var in self.workers if var not in new_workers]
                for dele in deles:
                    self.hashring.remove_node(dele)
                self.workers=new_workers
            self.time=ctime
        
        
    def pushurl(self,url):
        #TODO:rs push failed!
        node=self.hashring.get_node(url)
        self.rs.lpush(node,url)
        if node not in self.nodes.keys():
            self.nodes[node]=0
        self.nodes[node]+=1
        return node

    def get_nodes(self,):
        for node in self.nodes.keys():
            print node+":"+str(self.nodes[node])

    def run(self,):
        self.inter.start()
        while self.inter.running:
            ifile = self.inter.get_task()
            if ifile == None:
		time.sleep(3)
		continue
            if not os.path.exists(ifile):
		time.sleep(3)
		continue

            try:
		for line in open(ifile):
                    self.reload_worker()
                    if line.strip()!="":
                        node=self.pushurl(line.strip())
                    if self.inter.running == 0:
                        break

                self.get_nodes()
            except Exception,e:
                #TODO:write to log
                print str(e)
		continue