Python StrictRedis.zscore方法代码示例

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zscore [as 别名]
class GistStorage(object):
    SUPER_KEY = 'live'

    def __init__(self, host, port, password):
        self.client = StrictRedis(host=host, port=port, password=password)

    def upvote(self, gist_id):
        return self.client.zincrby(self.SUPER_KEY, gist_id, 1)

    def get_rank(self, gist_id):
        return self.client.zrevrank(self.SUPER_KEY, gist_id)

    def get_points(self, gist_id):
        return self.client.zscore(self.SUPER_KEY, gist_id)

    def get_rank_list(self, offset=0, num=-1):
        return self.client.zrevrange(self.SUPER_KEY, offset, num, withscores=True, score_cast_func=float)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zscore [as 别名]
class RedisDupeFilter(BaseDupeFilter):
    
    def __init__(self, path=None):
        self.redis_cli = StrictRedis(host='localhost', port=6379, db=0)
    
    @classmethod
    def from_settings(cls, settings):
        return cls(job_dir(settings))
     
    def request_seen(self, request):
        fp = request_fingerprint(request)
        domain = request.url.split('/')[2]
        suffix = 'category'
        if request.meta.has_key('skip'):
            suffix = 'content'
        key = '%s_%s' % (domain, suffix)
        timestamp = self.redis_cli.zscore(key, fp)
        if timestamp:
            #log.msg('skip %s' % request.url, level=log.INFO)
            return True
        self.redis_cli.zadd(key, time(), fp)
        return False

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zscore [as 别名]
class KVDB(object):
    """ A wrapper around the Zato's key-value database.
    """
    def __init__(self, conn=None, config=None, decrypt_func=None):
        self.conn = conn
        self.config = config
        self.decrypt_func = decrypt_func

    def init(self):
        config = {}

        if self.config.get('host'):
            config['host'] = self.config.host

        if self.config.get('port'):
            config['port'] = int(self.config.port)

        if self.config.get('db'):
            config['db'] = int(self.config.db)

        if self.config.get('password'):
            config['password'] = self.decrypt_func(self.config.password)

        if self.config.get('socket_timeout'):
            config['socket_timeout'] = float(self.config.socket_timeout)

        if self.config.get('connection_pool'):
            split = self.config.connection_pool.split('.')
            module, class_name = split[:-1], split[-1]
            mod = import_module(module)
            config['connection_pool'] = getattr(mod, class_name)

        if self.config.get('charset'):
            config['charset'] = self.config.charset

        if self.config.get('errors'):
            config['errors'] = self.config.errors

        if self.config.get('unix_socket_path'):
            config['unix_socket_path'] = self.config.unix_socket_path

        self.conn = StrictRedis(**config)

    def pubsub(self):
        return self.conn.pubsub()

    def publish(self, *args, **kwargs):
        return self.conn.publish(*args, **kwargs)

    def subscribe(self, *args, **kwargs):
        return self.conn.subscribe(*args, **kwargs)

    def translate(self, system1, key1, value1, system2, key2, default=''):
        return self.conn.hget(
            _KVDB.SEPARATOR.join(
                (_KVDB.TRANSLATION, system1, key1, value1, system2, key2)), 'value2') or default

    def copy(self):
        """ Returns an KVDB with the configuration copied over from self. Note that
        the object returned isn't initialized, in particular, the connection to the
        database won't have been initialized.
        """
        kvdb = KVDB()
        kvdb.config = self.config
        kvdb.decrypt_func = self.decrypt_func

        return kvdb

    def close(self):
        self.conn.connection_pool.disconnect()

# ##############################################################################

    # OAuth

    def add_oauth_nonce(self, username, nonce, max_nonce_log):
        """ Adds an OAuth to the set containing last N used ones for a given username.
        """
        key = NONCE_STORE.KEY_PATTERN.format('oauth', username)

        # This lets us trim the set to top (last) N nonces
        score = timegm(gmtime())

        self.conn.zadd(key, score, nonce)
        self.conn.zremrangebyrank(key, 0, -max_nonce_log)

    def has_oauth_nonce(self, username, nonce):
        """ Returns a boolean flag indicating if there's an OAuth nonce for a given
        username stored in KVDB.
        """
        return self.conn.zscore(NONCE_STORE.KEY_PATTERN.format('oauth', username), nonce)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zscore [as 别名]
class RedisTopTalkerTracker(object):
    def __init__(self, size=16384, redis_host='localhost', redis_port=6379,
                 redis_table='top_talkers'):
        self.size = size
        self.redis_table = 'top_talkers'
        self.client = StrictRedis(host=redis_host, port=redis_port)

        self.is_saturated = self.decide_is_saturated()

    def decide_is_saturated(self):
        return self.client.zcard(self.redis_table) >= self.size

    def clear(self):
        self.client.zremrangebyrank(self.redis_table, 0, -1)
        self.is_saturated = self.decide_is_saturated()

    def is_full(self):
        if self.is_saturated:
            return True

        self.is_saturated = self.decide_is_saturated()
        return self.is_saturated

    def get(self, key):
        count = self.client.zscore(self.redis_table, key)
        if count is None:
            return count

        return int(count)

    def contains(self, key):
        count = self.client.zscore(self.redis_table, key)
        return count is not None

    def add(self, key):
        # If it's already in there, increment its count and we're done.
        count = self.client.zscore(self.redis_table, key)
        if count is not None:
            self.client.zincrby(self.redis_table, key, 1)
            return

        # Else if the key is new to us but we're full, pop the lowest key/count
        # pair and insert the new key as count + 1.
        if self.is_full():
            keys_counts = self.client.zrange(
                self.redis_table, 0, 0, withscores=True, score_cast_func=int)
            old_count = keys_counts[0][1]
            self.client.zremrangebyrank(self.redis_table, 0, 0)
            new_count = old_count + 1
            self.client.zadd(self.redis_table, new_count, key)
            return

        # Or if the key is new to us and we have space, just insert it.
        self.client.zadd(self.redis_table, 1, key)

    def top_n_keys(self, n):
        return self.client.zrevrange(
            self.redis_table, 0, n - 1, score_cast_func=int)

    def top_n_keys_counts(self, n):
        keys_counts = self.client.zrevrange(
            self.redis_table, 0, n - 1, withscores=True, score_cast_func=int)
        if self.is_full():
            lowest_keys_counts = self.client.zrange(
                self.redis_table, 0, 0, withscores=True, score_cast_func=int)
            the_min = lowest_keys_counts[0][1] - 1
        else:
            the_min = 0
        return map(lambda (key, count): (key, count - the_min), keys_counts)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import zscore [as 别名]
class Queue(object):
    """
    Implements a sort-of-queue of crawl tasks.
    """
    def __init__(self, name, url_canonicalizer=None):
        self.name = name
        self.canonicalize_url = url_canonicalizer or (lambda url: url)
        self.redis = StrictRedis(host='localhost', port=6379, db=0)
        # XXX Temporary
        #self.redis.flushall()

    def serialize(self, task):
        return pickle.dumps(task)

    def deserialize(self, s):
        return pickle.loads(s)

    def prefix_redis_key(self, prefix, key):
        return ':'.join([self.name, prefix, key])

    def pick_queue_key(self, high_priority):
        return self.prefix_redis_key('todo', 'hp' if high_priority else 'nn')

    def hash_url(self, url):
        return hashlib.md5(url).hexdigest()

    def record_crawl_timestamp(self, url, now):
        self.redis.set(self.prefix_redis_key('urlts', self.hash_url(url)),
                       '%d' % now)

    def get_crawl_timestamp(self, url):
        s = self.redis.get(self.prefix_redis_key('urlts', self.hash_url(url)))
        if s:
            return int(s)

    def get_existing_task_id(self, url):
        s = self.redis.get(self.prefix_redis_key('taskbyurl',
                                                 self.hash_url(url)))
        if s:
            task_id, high_priority_bool = s.split(',')
            return task_id, high_priority_bool == '1'
        else:
            return None, None

    def get_task(self, task_id):
        s = self.redis.get(task_id)
        return self.deserialize(s)

    def drop_task(self, task_id, high_priority):
        self.redis.zrem(self.pick_queue_key(high_priority), task_id)
        self.redis.delete(task_id)

    def push(self, task):
        """
        Schedule a new crawl task, which is an instance of ``Task``.

        If ``high_priority`` is set, drop all other tasks for the same
        canonicalized URL and enqueue to a separate high priority queue.

        If ``min_age`` is set and this task URL has been crawled before, check
        the last time it was crawled, and reschedule it to a timestamp which is
        ``min_age`` after the last crawl time.

        If a different crawl task is already scheduled for this URL, check the
        scheduled time. If it has an intentionally delayed crawl time and that
        scheduled time is *after* this task, drop it. Otherwise, drop this
        task. Either way, the earlier of the two tasks should be kept.
        """
        # XXX FIXME
        # We really should take a lock on this URL, to prevent race conditions
        # in the conditional logic below.

        existing_task_id, existing_is_hp = self.get_existing_task_id(task.url)

        if task.high_priority:
            # If this task is HP, drop any other task for this URL..
            if existing_task_id:
                self.drop_task(existing_task_id, existing_is_hp)

        elif existing_is_hp:
            # If there's an existing high priority task, drop this one.
            return

        else:
            # If min_age is set, check for previous crawls and further delay
            # this task in order to ensure that min_age is not violated.
            if task.min_age:
                last_timestamp = self.get_crawl_timestamp(task.url)
                if last_timestamp:
                    task.scheduled_timestamp = max(last_timestamp +
                                                   task.min_age,
                                                   task.scheduled_timestamp)

            # If there is an existing task, check if it's scheduled earlier.
            if existing_task_id:
                scheduled_ts = self.redis.zscore(
                    self.pick_queue_key(existing_is_hp),
                    existing_task_id)

                if scheduled_ts > task.scheduled_timestamp:
#.........这里部分代码省略.........