Python Logger.info方法代码示例

# 需要导入模块: from kademlia.log import Logger [as 别名]
# 或者: from kademlia.log.Logger import info [as 别名]
class SpiderCrawl(object):
    """
    Crawl the network and look for given 160-bit keys.
    """
    def __init__(self, protocol, node, peers, ksize, alpha):
        """
        Create a new C{SpiderCrawl}er.

        Args:
            protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance.
            node: A :class:`~kademlia.node.Node` representing the key we're looking for
            peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network
            ksize: The value for k based on the paper
            alpha: The value for alpha based on the paper
        """
        self.protocol = protocol
        self.ksize = ksize
        self.alpha = alpha
        self.node = node
        self.nearest = NodeHeap(self.node, self.ksize)
        self.lastIDsCrawled = []
        self.log = Logger(system=self)
        self.log.info("creating spider with peers: %s" % peers)
        self.nearest.push(peers)

    def onError(self, err):
        self.log.error(repr(err))
        return err

    def _find(self, rpcmethod):
        """
        Get either a value or list of nodes.

        Args:
            rpcmethod: The protocol's callfindValue or callFindNode.

        The process:
          1. calls find_* to current ALPHA nearest not already queried nodes,
             adding results to current nearest list of k nodes.
          2. current nearest list needs to keep track of who has been queried already
             sort by nearest, keep KSIZE
          3. if list is same as last time, next call should be to everyone not
             yet queried
          4. repeat, unless nearest list has all been queried, then ur done
        """
        self.log.info("crawling with nearest: %s" % str(tuple(self.nearest)))
        count = self.alpha
        if self.nearest.getIDs() == self.lastIDsCrawled:
            self.log.info("last iteration same as current - checking all in list now")
            count = len(self.nearest)
        self.lastIDsCrawled = self.nearest.getIDs()

        ds = {}
        for peer in self.nearest.getUncontacted()[:count]:
            ds[peer.id] = rpcmethod(peer, self.node)
            self.nearest.markContacted(peer)
        d = deferredDict(ds)
        d.addCallback(self._nodesFound)
        d.addErrback(self.onError)
        return d

# 需要导入模块: from kademlia.log import Logger [as 别名]
# 或者: from kademlia.log.Logger import info [as 别名]
class KademliaProtocol(RPCProtocol):
    def __init__(self, sourceNode, storage, ksize):
        RPCProtocol.__init__(self)
        self.router = RoutingTable(self, ksize, sourceNode)
        self.storage = storage
        self.sourceNode = sourceNode
        self.log = Logger(system=self)

    def getRefreshIDs(self):
        """
        Get ids to search for to keep old buckets up to date.
        """
        ids = []
        for bucket in self.router.getLonelyBuckets():
            ids.append(random.randint(*bucket.range))
        return ids

    def rpc_stun(self, sender):
        return sender

    def rpc_ping(self, sender, nodeid):
        source = Node(nodeid, sender[0], sender[1])
        self.router.addContact(source)
        return self.sourceNode.id

    def rpc_store(self, sender, nodeid, key, value):
        source = Node(nodeid, sender[0], sender[1])
        self.router.addContact(source)
        self.log.debug("got a store request from %s, storing value" % str(sender))
        self.storage[key] = value
        return True

    def rpc_find_node(self, sender, nodeid, key):
        self.log.info("finding neighbors of %i in local table" % long(nodeid.encode('hex'), 16))
        source = Node(nodeid, sender[0], sender[1])
        self.router.addContact(source)
        node = Node(key)
        return map(tuple, self.router.findNeighbors(node, exclude=source))

    def rpc_find_value(self, sender, nodeid, key):
        source = Node(nodeid, sender[0], sender[1])
        self.router.addContact(source)
        value = self.storage.get(key, None)
        if value is None:
            return self.rpc_find_node(sender, nodeid, key)
        return { 'value': value }

    def callFindNode(self, nodeToAsk, nodeToFind):
        address = (nodeToAsk.ip, nodeToAsk.port)
        d = self.find_node(address, self.sourceNode.id, nodeToFind.id)
        return d.addCallback(self.handleCallResponse, nodeToAsk)

    def callFindValue(self, nodeToAsk, nodeToFind):
        address = (nodeToAsk.ip, nodeToAsk.port)
        d = self.find_value(address, self.sourceNode.id, nodeToFind.id)
        return d.addCallback(self.handleCallResponse, nodeToAsk)

    def callPing(self, nodeToAsk):
        address = (nodeToAsk.ip, nodeToAsk.port)
        d = self.ping(address, self.sourceNode.id)
        return d.addCallback(self.handleCallResponse, nodeToAsk)

    def callStore(self, nodeToAsk, key, value):
        address = (nodeToAsk.ip, nodeToAsk.port)
        d = self.store(address, self.sourceNode.id, key, value)
        return d.addCallback(self.handleCallResponse, nodeToAsk)

    def transferKeyValues(self, node):
        """
        Given a new node, send it all the keys/values it should be storing.

        @param node: A new node that just joined (or that we just found out
        about).

        Process:
        For each key in storage, get k closest nodes.  If newnode is closer
        than the furtherst in that list, and the node for this server
        is closer than the closest in that list, then store the key/value
        on the new node (per section 2.5 of the paper)
        """
        ds = []
        for key, value in self.storage.iteritems():
            keynode = Node(digest(key))
            neighbors = self.router.findNeighbors(keynode)
            if len(neighbors) > 0:
                newNodeClose = node.distanceTo(keynode) < neighbors[-1].distanceTo(keynode)
                thisNodeClosest = self.sourceNode.distanceTo(keynode) < neighbors[0].distanceTo(keynode)
            if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
                ds.append(self.callStore(node, key, value))
        return defer.gatherResults(ds)

    def handleCallResponse(self, result, node):
        """
        If we get a response, add the node to the routing table.  If
        we get no response, make sure it's removed from the routing table.
        """
        if result[0]:
            self.log.info("got response from %s, adding to router" % node)
            self.router.addContact(node)
            if self.router.isNewNode(node):
#.........这里部分代码省略.........

# 需要导入模块: from kademlia.log import Logger [as 别名]
# 或者: from kademlia.log.Logger import info [as 别名]

#.........这里部分代码省略.........

    def inetVisibleIP(self):
        """
        Get the internet visible IP's of this node as other nodes see it.

        @return: An C{list} of IP's.  If no one can be contacted, then the
        C{list} will be empty.
        """
        def handle(results):
            ips = [ result[1][0] for result in results if result[0] ]
            self.log.debug("other nodes think our ip is %s" % str(ips))
            return ips

        ds = []
        for neighbor in self.bootstrappableNeighbors():
            ds.append(self.protocol.stun(neighbor))
        return defer.gatherResults(ds).addCallback(handle)

    def get(self, key):
        """
        Get a key if the network has it.

        @return: C{None} if not found, the value otherwise.
        """
        node = Node(digest(key))
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to get key %s" % key)
            return defer.succeed(None)
        spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find()

    def set(self, key, value):
        """
        Set the given key to the given value in the network.
        """
        self.log.debug("setting '%s' = '%s' on network" % (key, value))
        dkey = digest(key)

        def store(nodes):
            self.log.info("setting '%s' on %s" % (key, map(str, nodes)))
            ds = [self.protocol.callStore(node, dkey, value) for node in nodes]
            return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)

        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to set key %s" % key)
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(store)

    def _anyRespondSuccess(self, responses):
        """
        Given the result of a DeferredList of calls to peers, ensure that at least
        one of them was contacted and responded with a Truthy result.
        """
        for deferSuccess, result in responses:
            peerReached, peerResponse = result
            if deferSuccess and peerReached and peerResponse:
                return True
        return False

    def saveState(self, fname):
        """
        Save the state of this node (the alpha/ksize/id/immediate neighbors)
        to a cache file with the given fname.
        """
        data = { 'ksize': self.ksize,
                 'alpha': self.alpha,
                 'id': self.node.id,
                 'neighbors': self.bootstrappableNeighbors() }
        with open(fname, 'w') as f:
            pickle.dump(data, f)

    @classmethod
    def loadState(self, fname):
        """
        Load the state of this node (the alpha/ksize/id/immediate neighbors)
        from a cache file with the given fname.
        """        
        with open(fname, 'r') as f:
            data = pickle.load(f)
        s = Server(data['ksize'], data['alpha'], data['id'])
        if len(data['neighbors']) > 0:
            s.bootstrap(data['neighbors'])
        return s

    def saveStateRegularly(self, fname, frequency=600):
        """
        Save the state of node with a given regularity to the given
        filename.
        
        @param fname: File to save retularly to
        @param frequencey: Frequency in seconds that the state
        should be saved.  By default, 10 minutes.
        """
        loop = LoopingCall(self.saveState, fname)
        loop.start(frequency)
        return loop

# 需要导入模块: from kademlia.log import Logger [as 别名]
# 或者: from kademlia.log.Logger import info [as 别名]
class BlockStorage(object):
    implements(IStorage)

    """ BlockStorage has following properties:
        a) is content-addressable (all keys must be hash(value))
        b) high TTL (effectively the keys don't expire)
        c) stores only valid JSON values
    """

    def __init__(self, ttl=STORAGE_TTL):
        """
        By default, max age is three years.
        """
        self.data = OrderedDict()
        self.ttl = ttl
        self.log = Logger(system=self)

    def __setitem__(self, key, value):

        try:
            test_value = json.loads(value)
        except:
            self.log.info("value not JSON, not storing")
            return

        hash = pybitcoin.hash.hex_hash160(value)
        test_key = digest(hash)

        if key != test_key:
            self.log.info("hash(value) doesn't match, not storing")
            return

        if key in self.data:
            del self.data[key]

        self.data[key] = (time.time(), value)
        self.cull()

    def cull(self):
        for k, v in self.iteritemsOlderThan(self.ttl):
            self.data.popitem(first=True)

    def get(self, key, default=None):
        self.cull()
        if key in self.data:
            value = self[key]
            hash = pybitcoin.hash.hex_hash160(value)

            test_key = digest(hash)

            if key != test_key:
                self.log.info("hash(value) doesn't match, ignoring value")
                return default

            return self[key]

        return default

    def __getitem__(self, key):
        self.cull()
        return self.data[key][1]

    def __iter__(self):
        self.cull()
        return iter(self.data)

    def __repr__(self):
        self.cull()
        return repr(self.data)

    def iteritemsOlderThan(self, secondsOld):
        minBirthday = time.time() - secondsOld
        zipped = self._tripleIterable()
        matches = takewhile(lambda r: minBirthday >= r[1], zipped)
        return imap(operator.itemgetter(0, 2), matches)

    def _tripleIterable(self):
        ikeys = self.data.iterkeys()
        ibirthday = imap(operator.itemgetter(0), self.data.itervalues())
        ivalues = imap(operator.itemgetter(1), self.data.itervalues())
        return izip(ikeys, ibirthday, ivalues)

    def iteritems(self):
        self.cull()
        ikeys = self.data.iterkeys()
        ivalues = imap(operator.itemgetter(1), self.data.itervalues())
        return izip(ikeys, ivalues)

# 需要导入模块: from kademlia.log import Logger [as 别名]
# 或者: from kademlia.log.Logger import info [as 别名]
class DHTMirrorRPC(jsonrpc.JSONRPC):
    """ A DHT Mirror with faster get/set."""

    def _get_hash(self, value):

        if type(value) is not dict:
            try:
                #self.log.info("WARNING: converting to json")
                value = json.loads(value)
            except:
                self.log.info("WARNING: not valid json")

        return hex_hash160(json.dumps(value, sort_keys=True))

    def __init__(self, dht_server=None):
        self.dht_server = dht_server
        self.log = Logger(system=self)

    def jsonrpc_ping(self):

        reply = {}
        reply['status'] = "alive"
        return reply

    def jsonrpc_stats(self):
        stats = {}
        stats['entries'] = dht_mirror.count()
        return stats

    def jsonrpc_get(self, key):

        resp = {}
        resp['key'] = key

        self.log.info("Get request for key: %s" % key)

        entry = dht_mirror.find_one({"key": key})

        if entry is not None:
            resp['value'] = entry['value']
        else:
            # if not in mirror/cache get from DHT
            return self.jsonrpc_dht_get(key)

        return resp

    def jsonrpc_set(self, key, value):

        self.log.info("Set request for key: %s" % key)

        resp = {}

        test_hash = self._get_hash(value)

        if test_hash != key:
            resp['error'] = "hash(value) doesn't match key"
            return resp

        write_to_cache(key, value)

        # perform the dht set/refresh in the background
        self.jsonrpc_dht_set(key, value)

        resp['status'] = 'success'

        return resp

    def jsonrpc_dht_get(self, key):

        self.log.info("DHT get request for key: %s" % key)

        resp = {}

        try:
            resp = self.dht_server.get(key)
            value = resp[0]
            write_to_cache(key, value)

        except Exception as e:
            resp['error'] = e

        return resp

    def jsonrpc_dht_set(self, key, value):

        self.log.info("DHT set request for key: %s" % key)

        resp = {}

        try:
            resp = self.dht_server.set(key, value)
        except Exception as e:
            resp['error'] = e

        return resp

# 需要导入模块: from kademlia.log import Logger [as 别名]
# 或者: from kademlia.log.Logger import info [as 别名]
class NetworkInterface (object):

    # Create a NetworkInterface object to accomplish all network related tasks
    def __init__(self, appDeployer, uuid):
        self._connected = False
        self._app_deployer = appDeployer

        # optional...
        self._number_of_nodes = 0
        self._list_of_nodes =[] 
        
        # logging capabilities
        self._log = Logger(system=self)

        # HERE--> Implementation specific node instanciation
        from kademlia.network import Server
        self._node = Server()
        self._node.log.level = 4
        # END OF SECTION 

    def bootStrapDone(self, server):
        #contacts = self._node.inetVisibleIP()
        print "BOOOOTTTT STAPPP IT"

    def retrieveContacts(self):
        """
        NEED TO FIND A WAY TO RETRIEVE THE LIST OF NEIGHBORS !!!
        
        """
        # !!! DOES EXACTLY THE SAME AS bootstrappableNeighbors !!!
        for bucket in self._node.protocol.router.buckets:
            print bucket.getNodes()
        
        # !!! bootstrappableNeighbors returns only the list of neighbors that you provided as !!!
        # !!! a bootstrap list, that are also online !!!
        neighbors =  self._node.bootstrappableNeighbors()
        
        print neighbors
        return neighbors

    def connect(self,fromPort,toPort,ip='127.0.0.1'):
        self._log.debug('Connecting...')
        #print "in connect ... "  
        #print "now listening on port: ",fromPort
        self._node.listen(fromPort)
        return self._node.bootstrap([(ip,toPort)]).addCallback(self.bootStrapDone)
            
    # This function is used to set a value in the DHT
    def setDone(self,result):
        print result
        print "set is done"
        deferred = Deferred()
        return deferred

    def set(self, key, value):

        def _processKey(result, key, values):
            print result, key, values
            deferred = Deferred()
            # upon recovering the value of the key
            if result == None:
                deferred = self._node.set(key, values)
                return deferred 
                #.addCallback(self.setDone)
            else:
                for value in values:
                    if value not in result: 
                        # append + publish
                        result.append(value)
                    else:
                        self._log.info("Value is already in the corresponding key.")
                deferred = self._node.set(key, result)
            return deferred
            

        # Only application deployers are allowed to write to the DHT.
        if self._app_deployer != False: 
            deferred = Deferred()           
            # Two possible keys are allowed to be written to, the template key and their respective application key
            if ('template' == key or self._uuid == key) and key != None:
                # HERE --> Implementation Specific Code
                print  " :::  ", self,  " ::: ", key, " ::: ", value, " <----------------------------" 
                # if writing to the template, retrieve the value first then append to it if necessary
                if key == 'template':

                    deferred = self._node.get(key)
                    deferred.addCallback(_processKey, key, value)
                    return deferred
                    #self._node.set(key, value).addCallback(self.setDone)
                # END OF SECTION

        # Not Allowed to write to the DHT.
        else:
            self._log.info("Only application deployers are allowed to write values into the DHT!")
            

    def done(self,result):
        print "self: ", self
        print "Key result:", result

#.........这里部分代码省略.........