Python crawling.NodeSpiderCrawl类代码示例

    def resolve(self, guid):
        """
        Given a guid return a `Node` object containing its ip and port or none if it's
        not found.

        Args:
            guid: the 20 raw bytes representing the guid.
        """
        node_to_find = Node(guid)

        def check_for_node(nodes):
            for node in nodes:
                if node.id == node_to_find.id:
                    return node
            return None
        index = self.protocol.router.getBucketFor(node_to_find)
        nodes = self.protocol.router.buckets[index].getNodes()
        for node in nodes:
            if node.id == node_to_find.id:
                return defer.succeed(node)
        nearest = self.protocol.router.findNeighbors(node_to_find)
        if len(nearest) == 0:
            self.log.warning("there are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
            return defer.succeed(None)
        spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(check_for_node)

    def delete(self, keyword, key, signature):
        """
        Delete the given key/value pair from the keyword dictionary on the network.
        To delete you must provide a signature covering the key that you wish to
        delete. It will be verified against the public key stored in the value. We
        use our ksize as alpha to make sure we reach as many nodes storing our value
        as possible.

        Args:
            keyword: the `string` keyword where the data being deleted is stored.
            key: the 20 byte hash of the data.
            signature: a signature covering the key.

        """
        self.log.debug("deleting '%s':'%s' from the network" % (keyword.encode("hex"), key.encode("hex")))
        dkey = digest(keyword)

        def delete(nodes):
            self.log.debug("deleting '%s' on %s" % (key.encode("hex"), [str(i) for i in nodes]))
            ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes]

            if self.storage.getSpecific(dkey, key) is not None:
                self.storage.delete(dkey, key)

            return defer.DeferredList(ds).addCallback(_anyRespondSuccess)

        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("there are no known neighbors to delete key %s" % key.encode("hex"))
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize)
        return spider.find().addCallback(delete)

 def start_crawl(results):
     for node, result in results.items():
         if not result[0]:
             del self.nodes[(node.ip, node.port)]
     node = Node(digest(random.getrandbits(255)))
     nearest = self.kserver.protocol.router.findNeighbors(node)
     spider = NodeSpiderCrawl(self.kserver.protocol, node, nearest, 100, 4)
     spider.find().addCallback(gather_results)

    def refreshTable(self):
        """
        Refresh buckets that haven't had any lookups in the last hour
        (per section 2.3 of the paper).
        """
        ds = []
        for rid in self.protocol.getRefreshIDs():
            node = Node(rid)
            nearest = self.protocol.router.findNeighbors(node, self.alpha)
            spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
            ds.append(spider.find())

        def republishKeys(_):
            for bucket in self.protocol.router.buckets:
                for node in bucket.nodes.values():
                    self.protocol.transferKeyValues(node)

        return defer.gatherResults(ds).addCallback(republishKeys)

    def test_find(self):
        self._connecting_to_connected()
        self.wire_protocol[self.addr1] = self.con
        self.wire_protocol[self.addr2] = self.con
        self.wire_protocol[self.addr3] = self.con

        self.protocol.router.addContact(self.node1)
        self.protocol.router.addContact(self.node2)
        self.protocol.router.addContact(self.node3)

        node = Node(digest("s"))
        nearest = self.protocol.router.findNeighbors(node)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
        spider.find()

        self.clock.advance(constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)

    def test_nodesFound(self):
        self._connecting_to_connected()
        self.wire_protocol[self.addr1] = self.con
        self.wire_protocol[self.addr2] = self.con
        self.wire_protocol[self.addr3] = self.con

        self.protocol.router.addContact(self.node1)
        self.protocol.router.addContact(self.node2)
        self.protocol.router.addContact(self.node3)

        node = Node(digest("s"))
        nearest = self.protocol.router.findNeighbors(node)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
        response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
                           self.node3.getProto().SerializeToString()))
        responses = {self.node1.id: response}
        spider._nodesFound(responses)

        self.clock.advance(constants.PACKET_TIMEOUT)
        connection.REACTOR.runUntilCurrent()
        self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)

        response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
                           self.node3.getProto().SerializeToString()))
        responses = {self.node1.id: response}
        nodes = spider._nodesFound(responses)
        node_protos = []
        for n in nodes:
            node_protos.append(n.getProto())

        self.assertTrue(self.node1.getProto() in node_protos)
        self.assertTrue(self.node2.getProto() in node_protos)
        self.assertTrue(self.node3.getProto() in node_protos)

        response = (False, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
                            self.node3.getProto().SerializeToString()))
        responses = {self.node1.id: response}
        nodes = spider._nodesFound(responses)
        node_protos = []
        for n in nodes:
            node_protos.append(n.getProto())

        self.assertTrue(self.node2.getProto() in node_protos)
        self.assertTrue(self.node3.getProto() in node_protos)

 def initTable(results):
     nodes = []
     for addr, result in results.items():
         if result[0]:
             n = objects.Node()
             try:
                 n.ParseFromString(result[1][0])
                 pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:]
                 verify_key = nacl.signing.VerifyKey(pubkey)
                 verify_key.verify(n.signedPublicKey)
                 h = nacl.hash.sha512(n.signedPublicKey)
                 hash_pow = h[64:128]
                 if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
                     raise Exception('Invalid GUID')
                 nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey))
             except Exception:
                 self.log.warning("bootstrap node returned invalid GUID")
     spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
     return spider.find()

    def get_node(self, guid):
        node_to_find = Node(guid)

        def check_for_node(nodes):
            for node in nodes:
                if node.id == node_to_find.id:
                    return node
            return None
        index = self.protocol.router.getBucketFor(node_to_find)
        nodes = self.protocol.router.buckets[index].getNodes()
        for node in nodes:
            if node.id == node_to_find.id:
                return node
        nearest = self.protocol.router.findNeighbors(node_to_find)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
            return defer.succeed(None)
        spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
        spider.find().addCallback(check_for_node)

    def refreshTable(self):
        """
        Refresh buckets that haven't had any lookups in the last hour
        (per section 2.3 of the paper).
        """
        ds = []
        refresh_ids = self.protocol.getRefreshIDs()
        refresh_ids.append(digest(random.getrandbits(255)))  # random node so we get more diversity
        for rid in refresh_ids:
            node = Node(rid)
            nearest = self.protocol.router.findNeighbors(node, self.alpha)
            spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
            ds.append(spider.find())

        def republishKeys(_):
            self.log.debug("Republishing key/values...")
            neighbors = self.protocol.router.findNeighbors(self.node, exclude=self.node)
            for node in neighbors:
                self.protocol.transferKeyValues(node)

        return defer.gatherResults(ds).addCallback(republishKeys)

    def refreshTable(self):
        """
        Refresh buckets that haven't had any lookups in the last hour
        (per section 2.3 of the paper).
        """
        ds = []
        for id in self.protocol.getRefreshIDs():
            node = Node(id)
            nearest = self.protocol.router.findNeighbors(node, self.alpha)
            spider = NodeSpiderCrawl(self.protocol, node, nearest)
            ds.append(spider.find())

        def republishKeys(_):
            ds = []
            # Republish keys older than one hour
            for keyword in self.storage.iterkeys():
                for k, v in self.storage.iteritems(keyword):
                    if self.storage.get_ttl(keyword, k) < 601200:
                        ds.append(self.set(keyword, k, v))

        return defer.gatherResults(ds).addCallback(republishKeys)

    def set(self, keyword, key, value, ttl=604800):
        """
        Set the given key/value tuple at the hash of the given keyword.
        All values stored in the DHT are stored as dictionaries of key/value
        pairs. If a value already exists for a given keyword, the new key/value
        pair will be appended to the dictionary.

        Args:
            keyword: The keyword to use. Should be hashed with hash160 before
                passing it in here.
            key: the 20 byte hash of the data.
            value: a serialized `protos.objects.Node` object which serves as a
                pointer to the node storing the data.

        Return: True if at least one peer responded. False if the store rpc
            completely failed.
        """
        if len(keyword) != 20:
            return defer.succeed(False)

        self.log.debug("setting '%s' on network" % keyword.encode("hex"))

        def store(nodes):
            self.log.debug("setting '%s' on %s" % (keyword.encode("hex"), [str(i) for i in nodes]))
            ds = [self.protocol.callStore(node, keyword, key, value, ttl) for node in nodes]

            keynode = Node(keyword)
            if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
                self.storage[keyword] = (key, value, ttl)
                self.log.debug("got a store request from %s, storing value" % str(self.node))

            return defer.DeferredList(ds).addCallback(_anyRespondSuccess)

        node = Node(keyword)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("there are no known neighbors to set keyword %s" % keyword.encode("hex"))
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(store)

    def resolve(self, guid):
        """
        Given a guid return a `Node` object containing its ip and port or none if it's
        not found.

        Args:
            guid: the 20 raw bytes representing the guid.
        """
        self.log.debug("crawling dht to find IP for %s" % guid.encode("hex"))

        node_to_find = Node(guid)
        for connection in self.protocol.multiplexer.values():
            if connection.handler.node is not None and connection.handler.node.id == node_to_find.id:
                self.log.debug("%s successfully resolved as %s" % (guid.encode("hex"), connection.handler.node))
                return defer.succeed(connection.handler.node)

        def check_for_node(nodes):
            for node in nodes:
                if node.id == node_to_find.id:
                    self.log.debug("%s successfully resolved as %s" % (guid.encode("hex"), node))
                    return node
            self.log.debug("%s was not found in the dht" % guid.encode("hex"))
            return None

        index = self.protocol.router.getBucketFor(node_to_find)
        nodes = self.protocol.router.buckets[index].getNodes()
        for node in nodes:
            if node.id == node_to_find.id:
                self.log.debug("%s successfully resolved as %s" % (guid.encode("hex"), node))
                return defer.succeed(node)

        nearest = self.protocol.router.findNeighbors(node_to_find)
        if len(nearest) == 0:
            self.log.warning("there are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
            return defer.succeed(None)

        spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha, True)
        return spider.find().addCallback(check_for_node)

    def set(self, keyword, key, value):
        """
        Set the given key/value tuple at the hash of the given keyword.
        All values stored in the DHT are stored as dictionaries of key/value
        pairs. If a value already exists for a given keyword, the new key/value
        pair will be appended to the dictionary.

        Args:
            keyword: a `string` keyword. The SHA1 hash of which will be used as
                the key when inserting in the DHT.
            key: the 20 byte hash of the data.
            value: a serialized `protos.objects.Node` object which serves as a
                pointer to the node storing the data.

        Return: True if at least one peer responded. False if the store rpc
            completely failed.
        """
        self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value)))
        dkey = digest(keyword)

        def store(nodes):
            self.log.info("setting '%s' on %s" % (keyword, map(str, nodes)))
            ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes]

            keynode = Node(dkey)
            if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
                self.storage[dkey] = (key, value)
                self.log.debug("got a store request from %s, storing value" % str(self.node))

            return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)

        node = Node(dkey)
        nearest = self.protocol.router.findNeighbors(node)
        if len(nearest) == 0:
            self.log.warning("There are no known neighbors to set key %s" % key)
            return defer.succeed(False)
        spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
        return spider.find().addCallback(store)