Python DiskDict.pop方法代码示例

# 需要导入模块: from w3af.core.data.db.disk_dict import DiskDict [as 别名]
# 或者: from w3af.core.data.db.disk_dict.DiskDict import pop [as 别名]

#.........这里部分代码省略.........
    def _get_hash(self, item):
        if item is None or item == POISON_PILL:
            # Return ffff...ffff which is the latest (in alphanumeric order)
            # hash that exists in MD5. This forces the None item to be placed
            # at the end of the queue.
            #
            # Warning! If FuzzableRequest.get_hash() ever changes its
            # implementation this will stop working as expected!
            return self.LAST_MD5_HASH

        return item.get_hash()

    def _put(self, item):
        """
        Put a new item in the queue
        """
        #
        #   This is very useful information for finding bottlenecks in the
        #   framework / strategy
        #
        if len(self.memory) == self.max_in_memory:
            #
            #   If you see many messages like this in the scan log, then you
            #   might want to experiment with a larger maxsize for this queue
            #
            msg = ('OrderedCachedQueue.put() will write a %r item to the %s'
                   ' DiskDict. This uses more CPU and disk IO than storing'
                   ' in memory but will avoid high memory usage issues. The'
                   ' current %s DiskDict size is %s.')
            args = (self._get_class_name(item),
                    self.get_name(),
                    self.get_name(),
                    len(self.disk))
            om.out.debug(msg % args)

        #
        #   Get the item hash to store it in the queue order list, and insert
        #   it using bisect.insort() that will keep the order at a low cost
        #
        item_hash = self._get_hash(item)
        bisect.insort(self.queue_order, item_hash)

        #
        #   Keep an in-memory dict that allows us to find the fuzzable requests
        #   in the other dictionaries
        #
        unique_id = str(uuid.uuid4())

        unique_id_list = self.hash_to_uuid.setdefault(item_hash, [])
        bisect.insort(unique_id_list, unique_id)

        #
        #   And now we just save the item to memory (if there is space) or
        #   disk (if it doesn't fit on memory)
        #
        if len(self.memory) < self.max_in_memory:
            self.memory[unique_id] = item
        else:
            self.disk[unique_id] = item

        self._item_added_to_queue()

    def _get(self):
        """
        Get an item from the queue
        """
        item_hash = self.queue_order.pop(0)
        unique_id_list = self.hash_to_uuid.pop(item_hash)
        unique_id = unique_id_list.pop(0)

        if unique_id_list:
            #
            # There are still items in this unique_id_list, this is most likely
            # because two items with the same hash were added to the queue, and
            # only one of those has been read.
            #
            # Need to add the other item(s) to the list again
            #
            bisect.insort(self.queue_order, item_hash)
            self.hash_to_uuid[item_hash] = unique_id_list

        try:
            item = self.memory.pop(unique_id)
        except KeyError:
            item = self.disk.pop(unique_id)

            if len(self.disk):
                #
                #   If you see many messages like this in the scan log, then you
                #   might want to experiment with a larger maxsize for this queue
                #
                msg = ('OrderedCachedQueue.get() from %s DiskDict was used to'
                       ' read an item from disk. The current %s DiskDict'
                       ' size is %s.')
                args = (self.get_name(), self.get_name(), len(self.disk))
                om.out.debug(msg % args)

        self._item_left_queue()
        self.processed_tasks += 1
        return item

# 需要导入模块: from w3af.core.data.db.disk_dict import DiskDict [as 别名]
# 或者: from w3af.core.data.db.disk_dict.DiskDict import pop [as 别名]

#.........这里部分代码省略.........
    disk.

    The CacheQueue object implements these methods from QueueSpeedMeasurement:
        * get_input_rpm
        * get_output_rpm

    Which allows users to understand how fast a queue is moving.
    """
    def __init__(self, maxsize=0, name='Unknown'):
        self.name = name
        self.max_in_memory = maxsize

        QueueSpeedMeasurement.__init__(self)

        # We want to send zero to the maxsize of the Queue implementation
        # here because we can write an infinite number of items
        Queue.Queue.__init__(self, maxsize=0)

    def get_name(self):
        return self.name

    def next_item_saved_to_memory(self):
        return len(self.memory) < self.max_in_memory

    def _init(self, maxsize):
        """
        Initialize the dicts and pointer
        :param maxsize: The max size for the queue
        """
        self.memory = dict()
        self.disk = DiskDict(table_prefix='%sCachedQueue' % self.name)
        self.get_pointer = 0
        self.put_pointer = 0

    def _qsize(self, len=len):
        return len(self.memory) + len(self.disk)

    def _get_class_name(self, obj):
        try:
            return obj.__class__.__name__
        except:
            return type(obj)

    def _put(self, item):
        """
        Put a new item in the queue
        """
        #
        #   This is very useful information for finding bottlenecks in the
        #   framework / strategy
        #
        if len(self.memory) == self.max_in_memory:
            #
            #   If you see many messages like this in the scan log, then you
            #   might want to experiment with a larger maxsize for this queue
            #
            msg = ('CachedQueue.put() will write a %r item to the %s DiskDict.'
                   ' This uses more CPU and disk IO than storing in memory'
                   ' but will avoid high memory usage issues. The current'
                   ' %s DiskDict size is %s.')
            args = (self._get_class_name(item),
                    self.get_name(),
                    self.get_name(),
                    len(self.disk))
            om.out.debug(msg % args)

        #
        #   And now we just save the item to memory (if there is space) or
        #   disk (if it doesn't fit on memory)
        #
        if len(self.memory) < self.max_in_memory:
            self.memory[self.put_pointer] = item
        else:
            self.disk[self.put_pointer] = item

        self.put_pointer += 1
        self._item_added_to_queue()

    def _get(self):
        """
        Get an item from the queue
        """
        try:
            item = self.memory.pop(self.get_pointer)
        except KeyError:
            item = self.disk.pop(self.get_pointer)

            if len(self.disk):
                #
                #   If you see many messages like this in the scan log, then you
                #   might want to experiment with a larger maxsize for this queue
                #
                msg = ('CachedQueue.get() from %s DiskDict was used to read an'
                       ' item from disk. The current %s DiskDict size is %s.')
                args = (self.get_name(), self.get_name(), len(self.disk))
                om.out.debug(msg % args)

        self._item_left_queue()
        self.get_pointer += 1
        return item

# 需要导入模块: from w3af.core.data.db.disk_dict import DiskDict [as 别名]
# 或者: from w3af.core.data.db.disk_dict.DiskDict import pop [as 别名]

#.........这里部分代码省略.........
        :param key: A key
        :return: True if the key should be stored in memory
        """
        if key in self._get_keys_for_memory():
            return True

        return False

    def _increase_access_count(self, key):
        access_count = self._access_count.get(key, 0)
        access_count += 1
        self._access_count[key] = access_count

        self._move_key_to_disk_if_needed(key)
        self._move_key_to_memory_if_needed(key)

    def _move_key_to_disk_if_needed(self, key):
        """
        Analyzes the current access count for the last accessed key and
        checks if any if the keys in memory should be moved to disk.

        :param key: The key that was last accessed
        :return: The name of the key that was moved to disk, or None if
                 all the keys are still in memory.
        """
        for key in self._in_memory.keys():
            if not self._belongs_in_memory(key):
                try:
                    value = self._in_memory[key]
                except KeyError:
                    return None
                else:
                    self._disk_dict[key] = value
                    self._in_memory.pop(key, None)
                    return key

    def _move_key_to_memory_if_needed(self, key):
        """
        Analyzes the current access count for the last accessed key and
        checks if any if the keys in disk should be moved to memory.

        :param key: The key that was last accessed
        :return: The name of the key that was moved to memory, or None if
                 all the keys are still on disk.
        """
        key_belongs_in_memory = self._belongs_in_memory(key)

        if not key_belongs_in_memory:
            return None

        try:
            value = self._disk_dict[key]
        except KeyError:
            return None
        else:
            self._in_memory[key] = value
            self._disk_dict.pop(key, None)
            return key

    def __setitem__(self, key, value):
        if len(self._in_memory) < self._max_in_memory:
            self._in_memory[key] = value
        else:
            self._disk_dict[key] = value

        self._increase_access_count(key)