Python StrictRedis.zcard方法代码示例

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zcard [as 别名]
class WordList:
  def __init__(self):
    self.conn = Redis()
    self.CACHE_SIZE = 50
    self.CACHE_KEYS = "words-keys"
    self.CACHE_STORE = "words-store"
    self.WORD_FILE = os.path.join(os.path.expanduser("~"), '.words.txt')

  def _reorganize(self):
    pop_n = self.conn.zcard(self.CACHE_KEYS) - self.CACHE_SIZE
    if pop_n >= 0:
      to_pop = self.conn.zrange(self.CACHE_KEYS, 0, pop_n)
      #print pop_n, to_pop
      self.conn.zremrangebyrank(self.CACHE_KEYS, 0, pop_n)
      for k in to_pop:
        self.conn.hdel(self.CACHE_STORE, k)

  def _add_word(self, key, value):
    result = self.conn.hget(self.CACHE_STORE, key)
    if result:
      self.conn.zincrby(self.CACHE_KEYS, key, 1.0)
    else:
      self._reorganize()
      self.conn.hset(self.CACHE_STORE, key, value)
      self.conn.zadd(self.CACHE_KEYS, 1, key)

  def _get_words(self):
    try:
      words = self.conn.zrevrange(self.CACHE_KEYS, 0, -1, True)
      #hashs = self.conn.hgetall(self.CACHE_STORE)
      #print words
      #print hashs
      return words
    except redis.exceptions.ConnectionError:
      return None

  def dump_console(self):
    if os.path.isfile(self.WORD_FILE):
      with open(self.WORD_FILE, 'r') as f:
        print f.read()

  def write_file(self):
    words = self._get_words()
    if words is None:
      return
    content = '\n'.join(["%d. %s\t %d"%(i, x[0], int(x[1])) for i, x in enumerate(words)])
    with open(self.WORD_FILE, 'w+') as f:
      f.write(content)
      f.write('\n')

  def add_word(self, key):
    try:
      self._add_word(key,key)
      self.write_file()
    except redis.exceptions.ConnectionError:
      return

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zcard [as 别名]
class RedisTopTalkerTracker(object):
    def __init__(self, size=16384, redis_host='localhost', redis_port=6379,
                 redis_table='top_talkers'):
        self.size = size
        self.redis_table = 'top_talkers'
        self.client = StrictRedis(host=redis_host, port=redis_port)

        self.is_saturated = self.decide_is_saturated()

    def decide_is_saturated(self):
        return self.client.zcard(self.redis_table) >= self.size

    def clear(self):
        self.client.zremrangebyrank(self.redis_table, 0, -1)
        self.is_saturated = self.decide_is_saturated()

    def is_full(self):
        if self.is_saturated:
            return True

        self.is_saturated = self.decide_is_saturated()
        return self.is_saturated

    def get(self, key):
        count = self.client.zscore(self.redis_table, key)
        if count is None:
            return count

        return int(count)

    def contains(self, key):
        count = self.client.zscore(self.redis_table, key)
        return count is not None

    def add(self, key):
        # If it's already in there, increment its count and we're done.
        count = self.client.zscore(self.redis_table, key)
        if count is not None:
            self.client.zincrby(self.redis_table, key, 1)
            return

        # Else if the key is new to us but we're full, pop the lowest key/count
        # pair and insert the new key as count + 1.
        if self.is_full():
            keys_counts = self.client.zrange(
                self.redis_table, 0, 0, withscores=True, score_cast_func=int)
            old_count = keys_counts[0][1]
            self.client.zremrangebyrank(self.redis_table, 0, 0)
            new_count = old_count + 1
            self.client.zadd(self.redis_table, new_count, key)
            return

        # Or if the key is new to us and we have space, just insert it.
        self.client.zadd(self.redis_table, 1, key)

    def top_n_keys(self, n):
        return self.client.zrevrange(
            self.redis_table, 0, n - 1, score_cast_func=int)

    def top_n_keys_counts(self, n):
        keys_counts = self.client.zrevrange(
            self.redis_table, 0, n - 1, withscores=True, score_cast_func=int)
        if self.is_full():
            lowest_keys_counts = self.client.zrange(
                self.redis_table, 0, 0, withscores=True, score_cast_func=int)
            the_min = lowest_keys_counts[0][1] - 1
        else:
            the_min = 0
        return map(lambda (key, count): (key, count - the_min), keys_counts)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zcard [as 别名]

#.........这里部分代码省略.........
        Ensure we are using a UNIX timestamp
        """
        if isinstance(ts, datetime):
            return (ts - datetime(1970, 1, 1)).total_seconds()
        else:
            return ts

    def _encode_value_with_time(self, value, ts):
        """
        Redis will rank members with identical scores lexigraphically. Often this is not
        what we want for a leaderboard. Using the timed_ties option, we will r the
        timestamp in the decimal part of the float score and thereby use it for tie-breaking.
        tie_oldest_wins controls whether older or newer timestamps get ranked higher.
        """
        if not ts:
            ts = time.time()
        else:
            ts = self._dt_to_ts(ts)
        if self.reverse == self.tie_oldest_wins:
            # invert the timestamp for proper ordering
            ts = 3000000000 - ts
        to_dec = 0.0000000001
        return float(value) + (ts * to_dec)

    def _decode_value_with_time(self, combo):
        from_dec = 10000000000
        value = int(combo)
        ts = (combo - value) * from_dec
        if self.reverse == self.tie_oldest_wins:
            ts = datetime.utcfromtimestamp(3000000000 - ts)
        return value, ts

    def _leaders_with_ranks(self, key, offset, end):
        total = self.r.zcard(key)
        l = self._range(key, offset, end)
        if self.timed_ties:
            l = [((m,) + self._decode_value_with_time(s)) for (m, s) in l]
        else:
            l = [(m, s, 0) for (m, s) in l]
        log.info(l)
        with_ranks = self._add_ranks(l, offset)
        return total, with_ranks

    def set_metric(self, user, value, ts=None):
        """
        Set a new peak value for this user, e.g. high score
        """
        if self.timed_ties:
            value = self._encode_value_with_time(value, ts)

        for r in self.ranges:
            key = self._board_key(r)
            self.r.zadd(key, value, user)
            if r != self.RANGE_ALLTIME:
                self.r.expire(key, r.expiration)

    def inc_metric(self, user, value, ts=None):
        """
        Increment the current value for this user, e.g. total earned
        """
        if ts:
            log.warn('inc_metric: timestamps not supported yet')

        for r in self.ranges:
            key = self._board_key(r)
            self.r.zincrby(key, user, value)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zcard [as 别名]

#.........这里部分代码省略.........
            return

        else:
            # If min_age is set, check for previous crawls and further delay
            # this task in order to ensure that min_age is not violated.
            if task.min_age:
                last_timestamp = self.get_crawl_timestamp(task.url)
                if last_timestamp:
                    task.scheduled_timestamp = max(last_timestamp +
                                                   task.min_age,
                                                   task.scheduled_timestamp)

            # If there is an existing task, check if it's scheduled earlier.
            if existing_task_id:
                scheduled_ts = self.redis.zscore(
                    self.pick_queue_key(existing_is_hp),
                    existing_task_id)

                if scheduled_ts > task.scheduled_timestamp:
                    # If the existing task is later, drop it.
                    self.drop_task(existing_task_id, existing_is_hp)
                else:
                    # If this task is later, drop it and keep the existing
                    # task.
                    return

        # At this point we're ready to enqueue.
        s = self.serialize(task)
        task_id = hashlib.md5(s).hexdigest()

        # Store the task by ID.
        self.redis.set(task_id, s)

        # Finally, enqueue the task ID to the appropriate sorted set.
        self.redis.zadd(self.pick_queue_key(task.high_priority),
                        float(task.scheduled_timestamp), task_id)

    def pop_queue(self, high_priority, cutoff):
        key = self.pick_queue_key(high_priority)
        with self.redis.pipeline() as pipe:
            while True:
                try:
                    pipe.watch(key)
                    elements = pipe.zrangebyscore(key, min=0, max=cutoff,
                                                  start=0, num=1)
                    if elements:
                        task_id = elements[0]
                        pipe.multi()
                        pipe.zrem(key, task_id)
                        pipe.execute()
                        return task_id
                    else:
                        return
                except WatchError:
                    continue

    def pop(self):
        """
        Get the next scheduled crawl task, or raise Empty() if there is nothing
        to do.
        """
        now = time.time()
        for high_priority in (True, False):
            next_task_id = self.pop_queue(high_priority, now)
            if next_task_id:
                break

        # Nothing to do?
        if not next_task_id:
            raise Empty()

        # Otherwise, fetch the task and deserialize it.
        s = self.redis.get(next_task_id)
        self.redis.delete(next_task_id)
        task = self.deserialize(s)

        # Record a crawl timestamp for this URL.
        self.record_crawl_timestamp(task.url, now)

        # If the task has a repeat interval, reschedule it.
        if task.repeat_after:
            repeat_task = Task(url=task.url,
                               document_type=task.document_type,
                               repeat_after=task.repeat_after,
                               scheduled_timestamp=now + task.repeat_after)
            self.push(repeat_task)

        return task

    def count(self):
        """
        Return the number of currently enqueued tasks.
        """
        counts = []
        for high_priority in (True, False):
            key = self.pick_queue_key(high_priority)
            counts.append(self.redis.zcard(key))

        return {'high': counts[0],
                'normal': counts[1]}

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zcard [as 别名]
class RedisBackend:

    colls_index_fmt = 'plumbca:' + dfconf['mark_version'] + ':collections:index'
    metadata_fmt = 'plumbca:' + dfconf['mark_version'] + ':metadata:timeline:{name}'
    inc_coll_cache_fmt = 'plumbca:' + dfconf['mark_version'] + ':cache:{name}'
    sorted_count_coll_cache_fmt = 'plumbca:' + dfconf['mark_version'] + \
                                  ':sorted:count:cache:{name}:{tagging}:{ts}'
    unique_count_coll_cache_fmt = 'plumbca:' + dfconf['mark_version'] + \
                                  ':unique:count:cache:{name}:{tagging}:{ts}'

    def __init__(self):
        self.rdb = StrictRedis(host=rdconf['host'], port=rdconf['port'],
                               db=rdconf['db'])
        self.version = dfconf['mark_version']

    def set_collection_index(self, name, instance):
        """ Set the collection info of instance to the backend.
        """
        key = self.colls_index_fmt
        v = instance.__class__.__name__
        self.rdb.hset(key, name, packb(v))

    def get_collection_index(self, name):
        """ Get the collection info from backend by name.
        """
        key = self.colls_index_fmt
        rv = self.rdb.hget(key, name)
        return [name, unpackb(rv)] if rv else None

    def get_collection_indexes(self):
        """ Get all of the collections info from backend.
        """
        key = self.colls_index_fmt
        rv = self.rdb.hgetall(key)
        if rv:
            return {name.decode("utf-8"): unpackb(info)
                        for name, info in rv.items()}

    def delete_collection_keys(self, coll, klass=''):
        """ Danger! This method will erasing all values store in the key that
        should be only use it when you really known what are you doing.

        It is good for the testing to clean up the environment.
        """
        md_key = self.metadata_fmt.format(name=coll.name)
        self.rdb.delete(md_key)

        if klass == 'IncreaseCollection':
            cache_key = self.inc_coll_cache_fmt.format(name=coll.name)
            self.rdb.delete(cache_key)

    def get_collection_length(self, coll, klass=''):
        if not klass:
            klass = coll.__class__.__name__

        rv = []
        md_key = self.metadata_fmt.format(name=coll.name)
        md_len = self.rdb.zcard(md_key)
        rv.append(md_len)
        # print('** TL -', self.rdb.zrange(md_key, 0, -1, withscores=True))

        if klass == 'IncreaseCollection':
            cache_key = self.inc_coll_cache_fmt.format(name=coll.name)
            cache_len = self.rdb.hlen(cache_key)
            # notice that the cache_len is the length of all the items in cache_key
            rv.append(cache_len)

        return rv

    def set_collection_metadata(self, coll, tagging, expts, ts, *args):
        """ Insert data to the metadata structure if timestamp data do not
        exists. Note that the metadata structure include two types, timeline
        and expire.

        :param coll: collection class
        :param tagging: specific tagging string
        :param ts: the timestamp of the data
        :param expts: the expired timestamp of the data
        """
        md_key = self.metadata_fmt.format(name=coll.name)
        # Ensure the item of the specific `ts` whether it's exists or not,
        element = self.rdb.zrangebyscore(md_key, ts, ts)

        if element:
            info = unpackb(element[0])
            if tagging in info:
                # the tagging info already exists then do nothings
                return
            info[tagging] = [expts] + list(args)
            # remove the md_key and update new value atomically
            p = self.rdb.pipeline()
            p.zremrangebyscore(md_key, ts, ts)
            p.zadd(md_key, ts, packb(info))
            p.execute()

        else:
            info = {tagging: [expts] + list(args)}
            self.rdb.zadd(md_key, ts, packb(info))
        # print('-'*10)
        # print(tagging)
#.........这里部分代码省略.........

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zcard [as 别名]
class RedisLRU(object):
    def __init__(self, redis=None, **kwargs):
        if redis is not None:
            self._redis = redis
        else:
            self._redis = Redis(**kwargs)
        self.namespaces = {
            "default": 10000
        }

    def setup_namespace(self, namespace, size):
        """Set the LRU Size for a namespace.
        """
        self.namespaces[namespace] = int(size)

    def _serialize(self, s):
        # return json.dumps(s)
        return s

    def _unserialize(self, s):
        # s = s.decode("utf-8")
        # return json.loads(s)
        return s

    def _size(self, namespace):
        return self.namespaces[namespace]

    def _hit_store(self, namespace):
        if namespace not in self.namespaces:
            raise KeyError("invalid namespace")
        return "cache_keys_{}".format(namespace)

    def _value_store(self, namespace):
        if namespace not in self.namespaces:
            raise KeyError("invalid namespace")
        return "cache_values_{}".format(namespace)

    def _expire_old(self, namespace):
        hits = self._hit_store(namespace)
        size = self._size(namespace)
        count = self._redis.zcard(hits)
        if count >= size:
            values = self._value_store(namespace)
            items = self._redis.zrange(hits, 0, count-size)
            logger.error(items)
            self._redis.zremrangebyrank(hits, 0, count-size)
            self._redis.hdel(values, *items)

    def clear(self, namespace="default"):
        """Clear the Cache.
        """
        hits = self._hit_store(namespace)
        values = self._value_store(namespace)
        self._redis.delete(hits, values)

    def clearAll(self):
        """Clear all known namespaces.
        """
        for k in self.namespaces.iterkeys():
            self.clear(k)

    def store(self, key, value, namespace="default"):
        """Store a key value pair in cache.
        This will not update an existing item.
        """
        values = self._value_store(namespace)
        if not self._redis.hexists(values, key):
            hits = self._hit_store(namespace)
            self._expire_old(namespace)
            self._redis.hset(values, key, self._serialize(value))
            self._redis.zadd(hits, time.time(), key)
        else:
            hits = self._hit_store(namespace)
            self._redis.hset(values, key, self._serialize(value))
            self._redis.zadd(hits, time.time(), key)

    def get(self, key, namespace="default"):
        """Get a value from the cache.
        returns none if the key is not found.
        """
        values = self._value_store(namespace)
        value = self._redis.hget(values, key)
        if value:
            hits = self._hit_store(namespace)
            self._redis.zadd(hits, time.time(), key)
            return self._unserialize(value)
        return None

    def expire(self, key, namespace="default"):
        """Expire (invalidate) a key from the cache.
        """
        values = self._value_store(namespace)
        if self._redis.hexists(values, key):
            hits = self._hit_store(namespace)
            self._redis.hdel(values, key)
            self._redis.zrem(hits, key)