Python LRUCache.put方法代码示例

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class AdapterRegistry(object):
    """ Registry of adapters"""

    _sentinel = object()

    def __init__(self):
        self.underlying = adapter.AdapterRegistry()
        self.cache = LRUCache(500)

    def lookup_adapter(self, typ):
        """ Lookup adapter for ``typ``"""
        adapter = self.cache.get(typ, self._sentinel)
        if adapter is self._sentinel:
            adapter = self.underlying.lookup([typ], IJSONSerializeable, "")
            self.cache.put(typ, adapter)
        return adapter

    def register_adapter(self, typ, adapter=None):
        """ Register ``adapter`` for type ``typ``

        If no ``adapter`` supplied then this method returns decorator.
        """
        if adapter is None:
            def decorator(adapter):
                self.register_adapter_impl(typ, adapter)
                return adapter
            return decorator
        return self.register_adapter_impl(typ, adapter)

    def register_adapter_impl(self, typ, adapter):
        self.underlying.register(
            [implementedBy(typ)], IJSONSerializeable, "", adapter)
        self.cache.clear()

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class LRUCache(StorageInterface):
    """In memory LRU cache"""

    def __init__(self, max_size=1024):
        if max_size < 1:
            raise ValueError("max_size must be a positive integer greater than 0")
        self.max_size = max_size
        self.engine = LRUCacheEngine(max_size)

    def __getitem__(self, key):
        value = self.engine.get(key)
        if value is None:
            raise KeyError
        return value

    def __setitem__(self, key, value):
        self.engine.put(key, value)

    def __contains__(self, key):
        value = self.engine.get(key)
        if value is not None:
            return True
        else:
            return False

    def __len__(self):
        return self.max_size

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class StaticsMiddleware(object):
    def _adapt_path(self, path):
        return normcase(normpath(path))

    def __init__(self, app, root_dir, cache_max_age=3600):
        self.app = app
        self.cache_max_age = cache_max_age
        self.doc_root = self._adapt_path(root_dir)
        self.paths_cache = LRUCache(1024)

    def __call__(self, environ, start_response):
        full_path = environ['PATH_INFO']
        filepath = self.paths_cache.get(full_path)

        if filepath is None:
            path = full_path.split('/')
            if INVALID_PATH_PARTS(path):
                return HTTPNotFound('Out of bounds: %s' % environ['PATH_INFO'])(environ, start_response)
            filepath = self._adapt_path(join(self.doc_root, *path))
            self.paths_cache.put(full_path, filepath)

        if isfile(filepath):
            return FileServeApp(filepath, self.cache_max_age)(environ, start_response)

        return self.app(environ, start_response)

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class RMemorySessionStore(Singleton):
    def __init__(self, config):
        self._cache = LRUCache(config.session_cache_size)

    def push(self, token_id, data):
        self._cache.put(token_id, data)

    def get(self, token):
        return self._cache.get(token, None)

    def remove(self, token_id):
        try:
            self._cache.put(token_id, None)
        except KeyError:
            pass

    def contains(self, session_id):
        return self._cache.get(session_id) is not None

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class ClusterCache(object):
    def __init__(self, cache_size):
        self.lru = LRUCache(cache_size)
        self.hits = 0
        self.misses = 0

    def get(self, file_buffer, ptr):
        v = self.lru.get((file_buffer, ptr))
        if v is not None:
            self.hits += 1
            return v
        v = ClusterData(file_buffer, ptr)
        self.lru.put((file_buffer, ptr), v)
        self.misses += 1
        return v

    def clear(self):
        logger.debug("CACHE HITS " + str(self.hits) + " VS MISSES " + str(self.misses))
        self.lru.clear()

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class EDBag(Counter):
    def __init__(self):
        super(EDBag, self).__init__()
        self.cache1 = LRUCache(256) # values where distance=1
        self.cache2 = LRUCache(256) # values where distance>1

    def add(self, x):
        if not x in self:
            self.cache2.clear()
        self[x] += 1

    def closest_by_edit_distance(self, x):
        if x in self:
            # Optimization: if x is in multiset, then closest
            # edit dist = 0. Nothing can be any closer.
            return (x, 0)

        # Optimization: If we've looked up this value before, 
        # return previously computed answer.
        cached_answer = self.cache1.get(x)
        if cached_answer:
            return cached_answer
        cached_answer = self.cache2.get(x)
        if cached_answer:
            return cached_answer

        closest = None
        closest_dist = None
        for y,_ in self.most_common():
            d = editdistance.eval(x, y)
            if not closest_dist or d < closest_dist:
                closest = y
                closest_dist = d
                if d == 1:
                    # Optimization: nothing can be any closer, as
                    # we know there's nothing at edit distance 0 (x is not
                    # in the multiset).
                    self.cache1.put(x, (closest, closest_dist))
                    return (closest, closest_dist)

        self.cache2.put(x, (closest, closest_dist))
        return (closest, closest_dist)

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class LRUShelf(Shelf):
    """An in-memory Least-Recently Used shelf up to `maxsize`.."""

    def __init__(self, maxsize=1000):
        self.store = LRUCache(int(maxsize))

    def getitem(self, key):
        value = self.store.get(key, UNSET)
        if value is UNSET:
            raise KeyError(key)
        return value

    def setitem(self, key, value):
        self.store.put(key, value)

    def delitem(self, key):
        self.store.invalidate(key)

    def clear(self):
        self.store.clear()

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class RMemorySessionStore(RUtils.singleton.Singleton):
    def __init__(self):
        if hasattr(self, '_init'):
            return
        self._init = True
        self.config = RUtils.config.RConfig()
        self._cache = LRUCache(self.config.session_cache_size)

    def push(self, token_id, data):
        self._cache.put(token_id, data)

    def get(self, token):
        return self._cache.get(token, None)

    def remove(self, session_id):
        try:
            self._cache.put(session_id, None)
        except KeyError:
            pass

    def contains(self, session_id):
        return self._cache.get(session_id) is not None

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
def partly_distinct(iterable):
    """
    Filters items from iterable and **tries to return only distincts**.
    Keeps order.

    :param Iterable iterable: Something iterable we have to filter.

    >>> list(partly_distinct([1, 2, 3, 2, 1, 2, 3, 4]))
    ... [1, 2, 3, 4]

    .. note::
        Unlike :py:func:`distinct` it won't guarantee that all elements would
        be distinct. But if you have rather small cardinality of the stream,
        this would work.

    .. note::
        Current implementation guarantees support for 10000 distinct values.
        If your cardinality is bigger, there might be some duplicates.
    """
    cache = LRUCache(10000)
    for item in iterable:
        if not cache.get(item):
            cache.put(item, True)
            yield item

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]

#.........这里部分代码省略.........

    def unregister(self, provides, component, *requires, **kw):
        self._lkpcache.clear()
        name = kw.get('name', '')
        if name is ALL:
            del self.data[requires]
            return
        info = self.data.get(requires, {})
        del info[(provides, name)]
        all = info.get((provides, ALL), [])
        all.remove(component)
        if not all:
            del self.data[requires]

    def subscribe(self, fn, *requires, **kw):
        name = kw.get('name', '')
        if name is ALL:
            raise ValueError('ALL may not be used as a name to subscribe')
        newkw = {'name':name, 'default':_marker}
        subscribers = self.lookup(_subscribers, *requires, **newkw)
        if subscribers is _marker:
            subscribers = []
        subscribers.append(fn)
        self.register(_subscribers, subscribers, *requires, **kw)

    def unsubscribe(self, fn, *requires, **kw):
        name = kw.get('name', '')
        if name is ALL:
            raise ValueError('ALL may not be used as a name to unsubscribe')
        newkw = {'name':name, 'default':_marker}
        subscribers = self.lookup(_subscribers, *requires, **newkw)
        if subscribers is _marker:
            subscribers = []
        if fn in subscribers:
            subscribers.remove(fn)

    def notify(self, *objects, **kw):
        if not self.listener_registered:
            return # optimization
        subscribers = self.resolve(_subscribers, *objects, **kw)
        name = kw.get('name', '')
        if subscribers is not None:
            if name is ALL:
                for subscriberlist in subscribers:
                    for subscriber in subscriberlist:
                        subscriber(*objects)
            else:
                for subscriber in subscribers:
                    subscriber(*objects)

    def _lookup(self, provides, name, default, requires, default_requires):
        # the requires and default_requires arguments *must* be
        # hashable sequences of tuples composed of hashable objects
        reg = self.data

        cachekey = (provides, requires, name, default_requires)
        cached = self._lkpcache.get(cachekey, _marker)

        if cached is _marker:
            combinations = cached_augmented_product(requires, default_requires)
            regkey = (provides, name)
            for combo in combinations:
                try:
                    result = reg[combo][regkey]
                    self._lkpcache.put(cachekey, result)
                    return result
                except KeyError:
                    pass

            self._lkpcache.put(cachekey, _notfound)
            cached = _notfound
            
        if cached is _notfound:
            if default is _missing:
                raise LookupError(
                    "Couldn't find a component providing %s for requires "
                    "args %r with name `%s`" % (provides, list(requires), name))
            return default

        return cached

    def lookup(self, provides, *requires, **kw):
        req = []
        for val in requires:
            if not hasattr(val, '__iter__'):
                req.append((val,))
            else:
                req.append(tuple(val))
        name = kw.get('name', '')
        extras = ((None,),) * len(req)
        default = kw.get('default', _missing)
        return self._lookup(provides, name, default, tuple(req), extras)

    def resolve(self, provides, *objects, **kw):
        requires = tuple(
            [directlyprovidedby(obj)+alsoprovidedby(obj) for obj in objects ])
        extras = tuple([defaultprovidedby(obj) for obj in objects])
        name = kw.get('name', '')
        default = kw.get('default', _missing)
        return self._lookup(provides, name, default, requires, extras)

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class Filesystem:
    def __init__(self, basedir, notify_on_commit):
        self.basedir = basedir
        self._notify_on_commit = notify_on_commit
        self._changelog_cache = LRUCache(1000)  # is thread safe
        with self.get_sqlconn() as conn:
            row = conn.execute("select max(serial) from changelog").fetchone()
            serial = row[0]
            if serial is None:
                self.next_serial = 0
            else:
                self.next_serial = serial + 1
                # perform some crash recovery
                data = self.get_raw_changelog_entry(serial)
                changes, rel_renames = loads(data)
                check_pending_renames(str(self.basedir), rel_renames)

    def write_transaction(self, sqlconn):
        return FSWriter(self, sqlconn)

    def get_raw_changelog_entry(self, serial):
        q = "SELECT data FROM changelog WHERE serial = ?"
        with self.get_sqlconn() as conn:
            conn.text_factory = bytes
            row = conn.execute(q, (serial,)).fetchone()
            if row is not None:
                return bytes(row[0])
            return None

    def get_changes(self, serial):
        changes = self._changelog_cache.get(serial)
        if changes is None:
            data = self.get_raw_changelog_entry(serial)
            changes, rel_renames = loads(data)
            self._changelog_cache.put(serial, changes)
        return changes

    def cache_commit_changes(self, serial, changes):
        self._changelog_cache.put(serial, changes)

    def get_sqlconn(self):
        path = self.basedir.join(".sqlite")
        if not path.exists():
            with sqlite3.connect(str(path)) as conn:
                threadlog.info("DB: Creating schema")
                c = conn.cursor()
                c.execute("""
                    CREATE TABLE kv (
                        key TEXT NOT NULL PRIMARY KEY,
                        keyname TEXT,
                        serial INTEGER
                    )
                """)
                c.execute("""
                    CREATE TABLE changelog (
                        serial INTEGER PRIMARY KEY,
                        data BLOB NOT NULL
                    )
                """)
        conn = sqlite3.connect(str(path), timeout=60)
        return conn

    def db_read_typedkey(self, relpath, conn=None):
        new_conn = conn is None
        if new_conn:
            conn = self.get_sqlconn()
        q = "SELECT keyname, serial FROM kv WHERE key = ?"
        try:
            c = conn.cursor()
            row = c.execute(q, (relpath,)).fetchone()
            if row is None:
                raise KeyError(relpath)
            return tuple(row[:2])
        finally:
            if new_conn:
                conn.close()

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]

#.........这里部分代码省略.........
        if callable(value):
            spec = getargspec(value)
            if len(spec.args) == 1 and spec.args[0] == 'ctx':
                value = value(self)
            elif len(spec.args) == 1 and spec.args[0] == 'm':
                value = value(m)
            elif len(spec.args) == 2 and spec.args[0] == 'ctx' and spec.args[1] == 'm':
                value = value(self, m)
            else:
                raise ETLConfigurationException("Invalid lambda expression signature: %s" % spec.args)

        # If the value is not a string, it is immediately returned
        if not isinstance(value, str):
            return value

        # Process string values

        value = value.strip()

        pos = -1
        result = str(value)

        for dstart, dend in (('${|', '|}'), ('${', '}')):
            if (pos >= -1):
                pos = result.find(dstart)
            while (pos >= 0):
                pos_end = result.find(dend)
                expr = result[pos + len(dstart):pos_end].strip()

                compiled = self._compiled.get(expr)
                try:
                    if (not compiled):
                        compiled = compile(expr, '', 'eval')
                        self._compiled.put(expr, compiled)

                    c_locals = {"m": m, "ctx": self, "f": self.f, "props": self.props, "var": self.var, "cubetl": cubetl}
                    if data:
                        c_locals.update(data)
                    res = eval(compiled, self._globals, c_locals)

                    if (self.debug2):
                        if (isinstance(res, str)):
                            logger.debug('Evaluated: %s = %r' % (expr, res if (len(res) < 100) else res[:100] + ".."))
                        else:
                            logger.debug('Evaluated: %s = %r' % (expr, res))

                except (Exception) as e:
                    exc_type, exc_value, exc_traceback = sys.exc_info()

                    caller_component = None
                    frame = inspect.currentframe()
                    for caller in inspect.getouterframes(frame):
                        fc = Context._class_from_frame(caller[0])
                        if (isclass(fc) and issubclass(fc, Component)):
                            caller_component = caller[0].f_locals['self']
                            break

                    #logger.error("Error evaluating expression %s on data: %s" % (expr, m))
                    self._eval_error_message = m

                    logger.error('Error evaluating expression "%s" called from %s:\n%s' % (expr, caller_component, ("".join(traceback.format_exception_only(exc_type, exc_value)))))
                    raise

                if (pos > 0) or (pos_end < len(result) - (len(dend))):
                    result = result[0:pos] + str(res) + result[pos_end + (len(dend)):]
                    pos = result.find(dstart)

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class Context:
    def __init__(self):

        self.args = {}

        self.debug = False
        self.debug2 = False

        self.quiet = False

        self.config_files = []

        self.start_node = None
        self.start_message = {}

        self.props = {}

        self._globals = {"text": text, "cubetl": cubetl}

        self._compiled = LRUCache(512)  # TODO: Configurable

        self.comp = Components(self)

    @staticmethod
    def _class_from_frame(fr):
        try:
            class_type = fr.f_locals["self"].__class__
        except KeyError:
            class_type = None

        return class_type

    def interpolate(self, m, value, data={}):

        # TODO: Naive interpolation

        # TODO: We are enforcing unicode working around Python Spring seems to give strings, not unicode
        # This shall not be necessary and it's possibly bad practice
        pos = -1
        result = unicode(value)

        for dstart, dend in (("${|", "|}"), ("${", "}")):
            if pos >= -1:
                pos = result.find(dstart)
            while pos >= 0:
                pos_end = result.find(dend)
                expr = result[pos + len(dstart) : pos_end].strip()

                compiled = self._compiled.get(expr)
                try:
                    if not compiled:
                        compiled = compile(expr, "", "eval")
                        self._compiled.put(expr, compiled)

                    c_locals = {"m": m, "ctx": self, "cubetl": cubetl}
                    c_locals.update(data)
                    res = eval(compiled, self._globals, c_locals)

                    if self.debug2:
                        if isinstance(res, basestring):
                            logger.debug("Evaluated: %s = %r" % (expr, res if (len(res) < 100) else res[:100] + ".."))
                        else:
                            logger.debug("Evaluated: %s = %r" % (expr, res))

                except (Exception) as e:
                    exc_type, exc_value, exc_traceback = sys.exc_info()

                    caller_component = None
                    frame = inspect.currentframe()
                    for caller in inspect.getouterframes(frame):
                        fc = Context._class_from_frame(caller[0])
                        if isclass(fc) and issubclass(fc, Component):
                            caller_component = caller[0].f_locals["self"]
                            break

                    # logger.error("Error evaluating expression %s on data: %s" % (expr, m))
                    raise Exception(
                        'Error evaluating expression "%s" called from %s:\n%s'
                        % (expr, caller_component, ("".join(traceback.format_exception_only(exc_type, exc_value))))
                    )

                if (pos > 0) or (pos_end < len(result) - (len(dend))):
                    result = result[0:pos] + unicode(res) + result[pos_end + (len(dend)) :]
                    pos = result.find(dstart)
                else:
                    # Keep non-string types
                    result = res
                    pos = -2

        return result

    def copy_message(self, m):
        return copy.copy(m)

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]

#.........这里部分代码省略.........
                sender.send_message('Number of followed channels is more than 8. It may take a while to process.')
            self._subscribe(followed_channels, sender)

    def _unsub_all(self, value, sender):
        # unsubscribe all channels for sender
        # we can not send self._sublist directly, since unsub operates
        # self._sublist
        user_sub = copy.copy(self._sublist[sender])
        self._unsubscribe(user_sub, sender)

    def _subscribe(self, chs, sender):
        # Handles user request for subscribing channels
        # We actually let the LinotServant to follow these channels
        # so that we can check if they are online use streams/followed API

        # prompt a message to let user know i am still alive...
        sender.send_message('Processing ...')
        msg = io.BytesIO()

        not_found = []
        for ch in chs:
            check_name = ch.lower()
            # reduce api invocation
            if check_name in self._sublist[sender]:  # pragma: no cover
                continue
            ch_disp_name, stat = self._twitch.follow_channel(ch)
            if stat is False:
                not_found.append(ch)
            else:
                self._sublist_lock.acquire(True)
                self._sublist[sender].append(check_name)
                self._sublist_lock.release()
                self._channel_sub_count[check_name] += 1
                self._channel_name_cache.put(ch_disp_name.lower(), ch_disp_name)
                pickle.dump(self._sublist, open(self.SUB_FILE, 'wb+'))

        if len(not_found) > 0:
            print('Channel not found: ' + ' '.join(not_found), file=msg)
        print('Done', file=msg)
        sender.send_message(msg.getvalue())
        return

    def _unsubscribe(self, chs, sender):
        # prompt a message to let user know i am still alive...
        sender.send_message('Processing ...')
        msg = io.BytesIO()

        # Handles user request for unsubscribing channels
        not_found = []
        for ch in chs:
            check_name = ch.lower()
            self._sublist_lock.acquire(True)
            try:
                self._sublist[sender].remove(check_name)
            except ValueError:
                not_found.append(ch)
                self._sublist_lock.release()
                continue
            self._sublist_lock.release()
            self._channel_sub_count[check_name] -= 1
            if self._channel_sub_count[check_name] <= 0:
                # maybe we can try to not unfollow, so that we don't keep
                # generating follow message to the caster
                # self._twitch.unfollow_channel(ch)
                self._channel_sub_count.pop(check_name, None)

# 需要导入模块: from repoze.lru import LRUCache [as 别名]
# 或者: from repoze.lru.LRUCache import put [as 别名]
class CachingKeyLookup(object):
    """
    A key lookup that caches.

    Implements the read-only API of :class:`Registry`, using
    a cache to speed up access.

    The cache is LRU.

    :param: key_lookup - the :class:`Registry` to cache.
    :param component_cache_size: how many cache entries to store for
      the :meth:`component` method. This is also used by dispatch
      calls.
    :param all_cache_size: how many cache entries to store for the
      the :meth:`all` method.
    :param fallback_cache_size: how many cache entries to store for
      the :meth:`fallback` method.
    """
    def __init__(self, key_lookup, component_cache_size, all_cache_size,
                 fallback_cache_size):
        self.key_lookup = key_lookup
        self.predicate_key = key_lookup.predicate_key
        self.key_dict_to_predicate_key = key_lookup.key_dict_to_predicate_key
        self.component_cache = LRUCache(component_cache_size)
        self.all_cache = LRUCache(all_cache_size)
        self.fallback_cache = LRUCache(fallback_cache_size)

    def component(self, key, predicate_key):
        """Lookup value in registry based on predicate_key.

        If value for predicate_key cannot be found, looks up first
        permutation of predicate_key for which there is a value. Permutations
        are made according to the predicates registered for the key.

        :param key: an immutable for which to look up the predicate_key.
        :param predicate_key: an immutable predicate key, constructed
          for predicates given for this key.
        :returns: a registered value, or ``None``.
        """
        result = self.component_cache.get((key, predicate_key), NOT_FOUND)
        if result is not NOT_FOUND:
            return result
        result = self.key_lookup.component(key, predicate_key)
        self.component_cache.put((key, predicate_key), result)
        return result

    def fallback(self, key, predicate_key):
        """Lookup fallback based on predicate_key.

        This finds the fallback for the most specific predicate
        that fails to match.

        :param key: an immutable for which to look up the predicate_key.
        :param predicate_key: an immutable predicate key, constructed
          for predicates given for this key.
        :returns: the fallback value for the most specific predicate
          the failed to match.
        """
        result = self.fallback_cache.get((key, predicate_key), NOT_FOUND)
        if result is not NOT_FOUND:
            return result
        result = self.key_lookup.fallback(key, predicate_key)
        self.fallback_cache.put((key, predicate_key), result)
        return result

    def all(self, key, predicate_key):
        """Lookup iterable of values registered for predicate_key.

        Looks up values registered for all permutations of
        predicate_key, the most specific first.

        :param key: an immutable for which to look up the values.
        :param predicate_key: an immutable predicate key, constructed for
          the predicates given for this key.
        :returns: An iterable of registered values.
        """
        result = self.all_cache.get((key, predicate_key), NOT_FOUND)
        if result is not NOT_FOUND:
            return result
        result = list(self.key_lookup.all(key, predicate_key))
        self.all_cache.put((key, predicate_key), result)
        return result

    def lookup(self):
        """A :class:`Lookup` for this registry.
        """
        return Lookup(self)