Python StrictRedis.srandmember方法代码示例

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import srandmember [as 别名]
def random_dril_tweet_from_redis():
    """
    to counter-act the terrible "ISIS", im starting my own group called
    "NICEis". what we do is give retweets & faves to the
    hopelessly decrepit
    """
    redis_client = StrictRedis(host='localhost', port=6379, db=0)
    tweet = redis_client.srandmember('tweets')
    tweet = tweet.decode('UTF-8')
    return tweet

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import srandmember [as 别名]

#.........这里部分代码省略.........
            return []

        key = self.inc_coll_cache_fmt.format(name=coll.name)
        rv = self.rdb.hmget(key, *fields)
        # print('inc_coll_caches_get - ', rv)
        # print('inc_coll_caches_get After - ', [unpackb(r) for r in rv if r])
        return [unpackb(r) for r in rv if r]

    def inc_coll_caches_del(self, coll, *fields):
        key = self.inc_coll_cache_fmt.format(name=coll.name)
        return self.rdb.hdel(key, *fields)

    def uniq_count_coll_cache_set(self, coll, ts, tagging, values):
        """
        :param values: should be a iterable object contain members
        """
        values = {packb(v) for v in values}
        key_fmt = self.unique_count_coll_cache_fmt
        key = key_fmt.format(name=coll.name, tagging=tagging, ts=ts)
        return self.rdb.sadd(key, *values)

    def uniq_count_coll_cache_get(self, coll, tagging, timestamps, count_only=False):
        key_fmt = self.unique_count_coll_cache_fmt
        rv = []
        for ts in timestamps:
            key = key_fmt.format(name=coll.name, tagging=tagging, ts=ts)
            if count_only:
                count = self.rdb.scard(key)
                rv.append(count)
            else:
                members = self.rdb.smembers(key)
                rv.append({unpackb(m) for m in members})
        return rv

    def uniq_count_coll_cache_pop(self, coll, tagging, timestamps, number):
        """
        :note: Redis `SPOP key [count]` command, The count argument will be
               available in a later version and is not available
               in 2.6, 2.8, 3.0.
               Now use SRANDMEMBER and SREM commands to mimic the effect of
               SPOP count.
        """
        key_fmt = self.unique_count_coll_cache_fmt
        rv = []
        for ts in timestamps:
            key = key_fmt.format(name=coll.name, tagging=tagging, ts=ts)
            # :: srandmember + srem == spop(key, number)
            members = self.rdb.srandmember(key, number)
            self.rdb.srem(key, *members)
            rv.append({unpackb(m) for m in members})
        return rv

    def uniq_count_coll_cache_del(self, coll, tagging, timestamps):
        keys = self._gen_count_keys(coll.name, tagging,
                                    'unique_count', timestamps)
        return self.rdb.delete(*keys)

    def sorted_count_coll_cache_set(self, coll, ts, tagging, values):
        """
        :param values: should be a dict of <member: score> pair
        """
        key_fmt = self.sorted_count_coll_cache_fmt
        key = key_fmt.format(name=coll.name, tagging=tagging, ts=ts)
        add_val = []
        for member, score in values.items():
            add_val.append(score)
            add_val.append(packb(member))
        return self.rdb.zadd(key, *add_val)

    def sorted_count_coll_cache_get(self, coll, tagging, timestamps, topN=None):
        key_fmt = self.sorted_count_coll_cache_fmt
        rv = []
        for ts in timestamps:
            key = key_fmt.format(name=coll.name, tagging=tagging, ts=ts)
            if topN:
                elements = self.rdb.zrange(key, -topN, -1, withscores=True)
            else:
                elements = self.rdb.zrange(key, 0, -1, withscores=True)
            rv.append([(unpackb(member), score) for member, score in elements])
        # import pprint
        # pprint.pprint(rv)
        return rv

    def sorted_count_coll_cache_del(self, coll, tagging, timestamps):
        keys = self._gen_count_keys(coll.name, tagging,
                                    'sorted_count', timestamps)
        return self.rdb.delete(*keys)

    def _gen_count_keys(self, name, tagging, cachetype, timestamps):
        if cachetype == 'unique_count':
            key_fmt = self.unique_count_coll_cache_fmt
        elif cachetype == 'sorted_count':
            key_fmt = self.sorted_count_coll_cache_fmt

        keys = []
        for ts in timestamps:
            k = key_fmt.format(name=name, tagging=tagging, ts=ts)
            keys.append(k)

        return keys

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import srandmember [as 别名]

#.........这里部分代码省略.........
				minval = '(%d' % val
				maxval = '+inf'

			elif expr.operator == expr.GE:
				minval = val
				maxval = '+inf'

			elif expr.operator == expr.LT:
				minval = '-inf'
				maxval = '(%d' % val

			elif expr.operator == expr.LE:
				minval = '-inf'
				maxval = val

			else:
				raise Exception('Unsupported operator type given')

			start = expr.offset
			num = expr.limit

			if num is None and start == 0:
				start = None

			ids = self.handler.zrangebyscore(
				key,
				minval,
				maxval,
				start=start,
				num=num,
			)

			return [expr.model_cls(model_id) for model_id in ids]

	def choice (self, field, model_cls, val, count=1):
		key = self.idx_key(model_cls.getprefix(), field.name, val)
		ids = self.handler.srandmember(key, count)

		return None if not len(ids) else \
			[model_cls(model_id) for model_id in ids]

	def _save_idx (self, field, model, pipe=None):
		""" Save given model.field index. """

		if isinstance(field, IndexField):
			idx_key = self.idx_key(model.getprefix(), field.name, model[field.name])

			if field.unique:
				ids = self.handler.smembers(idx_key)

				if len(ids):
					ids.discard(bytes(model._id, 'utf-8') if PY3K else model._id)

					if len(ids):
						raise Exception('Duplicate key error')

			self._del_idx(field, model, pipe)
			pipe.sadd(idx_key, model._id)

		elif isinstance(field, RangeIndexField):
			val = model[field.name]

			if field.unique:
				models = field == val

				if len(models) > 1 or len(models) == 1 and models[0] is not model:
					raise Exception('Duplicate key error')

			key = self.ridx_key(model.getprefix(), field.name)

			pipe.zadd(key, **{
				model._id: field.to_db(val)
			})

		else:
			raise Exception('Bad field type given')

	def _del_idx (self, field, model, pipe=None):
		""" Delete db index value of model.field. """

		if model._exists is False:
			return

		if isinstance(field, IndexField):
			idx_val = model.get(
				name=field.name,
				origin=True,
				lite=True,
			)

			if idx_val is not None:
				idx_key = self.idx_key(model.getprefix(), field.name, idx_val)
				pipe.srem(idx_key, model._id)

		elif isinstance(field, RangeIndexField):
			key = self.ridx_key(model.getprefix(), field.name)
			pipe.zrem(key, model._id)

		else:
			raise Exception('Bad field type given')

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import srandmember [as 别名]
    return 0


def _proxy():
    if not rs.scard(rs_lname_proxy):
        time.sleep(1)
        _proxy()
    proxy = rs.srandmember('PROXY')
    return proxy


class SlowDownloadException(Exception):
    pass


user_agent = lambda: rs.srandmember('USER_AGENT')

_headers = {
    'User-Agent': user_agent(),
    # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
}


def _download(url, download_timeout=10):
    proxy = _proxy()
    _start = time.time()
    _auth = HTTPProxyAuth(username='dengta', password='sf123456')
    s = requests.session()
    req = s.get(url=url, timeout=3, stream=True, proxies={'http': proxy}, headers=_headers, auth=_auth)
    res = list()
    for chunk in req.iter_content(chunk_size=65535):