Python StrictRedis.sismember方法代码示例

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
def add_tweets_to_redis(tweet_file):
    """
    'DVD: FBI WARNING Me: oh boy here we go DVD: The board advises you
    to have lots of fun watching this Hollywood movie Me: Ah.. It's a
    nice one'
    """
    redis_client = StrictRedis(host='localhost', port=6379, db=0)
    with open(tweet_file, 'r') as tweets:
        for line in tweets:
            # again, dealing with weird error here
            try:
                tweet = line.strip().split('|', 2)[2]
                # need to investigate whether one-by-one inserting
                # or building a list of tweets and doing a single insert
                # would be more efficient
                if not redis_client.sismember('tweets', tweet):
                    result = redis_client.sadd('tweets', tweet)
                    if not result:
                        print('error occurred adding tweet: "{}" to redis'
                              .format(tweet))
                else:
                    # found all new tweets
                    break
            except IndexError:
                continue
    redis_client.save()

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
def check_auth(login, password, required_scope):
	"""This function is called to check if a login, password, and scope combination is valid. """

	# Hash the login, the password is ignored
	login_hash = calc_hash(login)
	cache = StrictRedis(db=config.tokens_cache_redis_db)	# TODO: Should be at application scope instead of request scope
	user = cache.get(login_hash)	# lookup our person
	#print('check_auth: login: ' + login + ' scope: '+ required_scope + ', user: ' + str(user))
	if (user is None):				# if the user has a valid token
		return False
	elif (required_scope == 'general'):
		return True
	elif (cache.sismember(required_scope, user) == True): 
		return True				# and the user has the required scope
	else:
		return False			# but the user does not have the required scope

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
def gdelete(request):
    groupname = request.POST.get('name', 0);
    rd = StrictRedis()
    pref = settings.MY_PREFIX
    prefg = pref+":"+groupname
    user = str(request.user)
    print "received request for deleting",groupname,"from",user
    ismember = rd.sismember(pref+":groups",groupname)
    if not ismember:
        return JsonResponse({'done':False,'reason':'No such group name'});
    # now check whether the requesting user is the one who created the group
    d = rd.hgetall(prefg+":hash")
    if d['owner'] != user:
        return JsonResponse({'done':False, 'reason':'Only group owner can delete the group'})
    rd.srem(pref+":groups", groupname)
    rd.delete(prefg+":hash", pref+":"+groupname)
    rd.delete(pref+":"+groupname)
    redis_publisher = RedisPublisher(facility = pref, broadcast = True)
    redis_publisher.publish_message(RedisMessage(json.dumps({"type":"group_delete", "name":groupname})))
    return JsonResponse({'done':True})

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class Redis(Backend):
    """Default backend for MimicDB. Initiated with identical parameters
    as ``redis.StrictRedis``.

    :param args \*args, \**kwargs: StrictRedis.__init__() parameters

    .. code-block:: python

        from mimicdb.backends.default import Redis

        redis = Redis(host='localhost', port=6379, db=0)
    """
    def __init__(self, *args, **kwargs):
        self._redis = StrictRedis(*args, **kwargs)

    def keys(self, pattern='*'):
        return self._redis.keys(pattern)

    def delete(self, *names):
        return self._redis.delete(*names)

    def sadd(self, name, *values):
        return self._redis.sadd(name, *values)

    def srem(self, name, *values):
        return self._redis.srem(name, *values)

    def sismember(self, name, value):
        return self._redis.sismember(name, value)

    def smembers(self, name):
        return self._redis.smembers(name)

    def hmset(self, name, mapping):
        return self._redis.hmset(name, mapping)

    def hgetall(self, name):
        return self._redis.hgetall(name)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]

#.........这里部分代码省略.........
        params = {'action': "sitematrix"}
        data = self.query(params)

        blacklist_wikis = ['loginwiki']
        flaggedrevs_wikis = self._flaggedrevs_wikis()

        wikis = {}
        for key, val in data['sitematrix'].items():
            if key == 'count':
                continue

            if 'code' in val:
                for site in val['site']:
                    wikis[site['dbname']] = self._create_wiki(
                        site, val['code'], val['name'],
                        flaggedrevs_wikis, blacklist_wikis)
            else:
                for site in val:
                    wikis[site['dbname']] = self._create_wiki(
                        site, '', '', flaggedrevs_wikis, blacklist_wikis)
        return wikis

    def _create_wiki(self, site, langcode, langname, flaggedrevs_wikis,
                     blacklist_wikis):
        wiki = {
            'lang': langcode,
            'langname': langname,
            'url': site['url'].replace("http://", "https://"),
            'dbname': site['dbname'],
            'group': site['code']
        }
        if wiki['group'] == 'wiki':
            wiki['group'] = 'wikipedia'

        inactive_codes = ['closed', 'private', 'fishbowl']
        if any([key in site for key in inactive_codes]):
            wiki['closed'] = True
        if site['dbname'] in blacklist_wikis:
            wiki['closed'] = True
        if site['dbname'] in flaggedrevs_wikis:
            wiki['flaggedrevs'] = True
        return wiki

    def _flaggedrevs_wikis(self):
        url = "https://noc.wikimedia.org/conf/flaggedrevs.dblist"
        response = requests.get(url, headers=self.headers)
        response.raise_for_status()

        return response.text.splitlines()

    def ores_context_exists(self, dbname, model='reverted', cached=True):
        """Checks if ORES context for a wiki exists"""
        key = config.redis_prefix + 'ores' + model
        if not self.redis.exists(key) or not cached:
            self._ores_contexts(model)
        return self.redis.sismember(key, dbname)

    def _ores_contexts(self, model):
        """Fill cache"""
        pipe = self.redis.pipeline()
        key = config.redis_prefix + 'ores' + model
        pipe.delete(key)

        contexts = requests.get(self.ores_url, headers=self.headers)
        contexts.raise_for_status()
        contexts = contexts.json()['contexts']

        for context in contexts:
            models = requests.get('{}/{}/'.format(self.ores_url, context),
                                  headers=self.headers)
            models.raise_for_status()
            models = models.json()['models']

            if model in models:
                pipe.sadd(key, context)
        pipe.expire(key, 172800)  # 2 days exp.
        pipe.execute()

    def ores_scores(self, dbname, revids, model='reverted'):
        """Get ORES scores for revision ids"""
        url = '{}/{}/{}/'.format(self.ores_url, dbname, model)

        revids = '|'.join([str(id) for id in revids])
        params = {'revids': revids}

        response = requests.get(url, params=params, headers=self.headers)

        if response.status_code != requests.codes.ok:
            raise Exception('ORES error code {} for {} with params {}'.format(
                response.status_code, dbname, str(params)), response.text)

        return response.json()

    def was_reverted(self, revision):
        """
        Checks if a revision was reverted
        :param revision: a revision dict containing ‘revid’ and ‘pageid’
        """
        session = RevertsSession(self.api_url, user_agent=self.user_agent)
        return reverts.check_rev(session, revision)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class Graph(object):
    """
        Main interface for adding/removing/looking up
        graph elements.
        
        basic usage:
        you'll add and remove elements via:
        -add_node()/add_edge()
        -del_node()/del_edge()
        
        you can also look up nodes by property:
        get_nodes()/get_edges()

        you can query by property with regex:
        find_nodes()/find_edges()

        all of these methods will return a list of 
        Node() or Edge() objects. From these, you are
        able to look at incoming or ourgoing edges, 
        parent and child nodes, and individually access
        and modify their properties.
        
        Usage::
            >>> from rgr import Graph
            >>> g = Graph()
            >>> g2 = Graph('mygraph')

        :param name: default -> 'rgr'; the namespace for your graph within redis.
    """
    #TODO: better errors
    def __init__(self, name='rgr'):
        self.redis = Redis()
        self.name = name
        self.next_nid_key = self.name + ':next_nid' #scalar
        self.next_eid_key = self.name + ':next_eid' #scalar
        self.nodes_key = self.name + ':nodes'       #set
        self.edges_key = self.name + ':edges'       #set
        if not self.redis.exists(self.next_nid_key):
            self.redis.set(self.next_nid_key, 0)
        if not self.redis.exists(self.next_eid_key):
            self.redis.set(self.next_eid_key, 0)

    def add_node(self, **kwargs):
        """
            Add a node to the graph.

            Usage::
                >>> from rgr import Graph
                >>> g = rgr.Graph()
                >>> n = g.add_node() #blank node
                >>> n2 = g.add_node(name='john', type='person')
        
            :param kwargs: node properties to initialize
            :return: rgr.Node representing the node you just created.
        """
        new_nid = self.redis.get(self.next_nid_key)
        new_node = Node(self, new_nid)
        self.redis.sadd(self.nodes_key, new_nid)
        for k in kwargs:
            new_node.prop.__setattr__(k, kwargs[k])
            self._index(new_node.name, k, kwargs[k]) 
        self.redis.incr(self.next_nid_key)
        return new_node

    def add_edge(self, parent, child, **kwargs):
        """
            Add an edge between two nodes.
            
            Usage::
                
                >>> from rgr import Graph
                >>> g = Graph()
                >>> john = g.add_node(name='john')
                >>> mary = g.add_node(name='mary')
                >>> e = g.add_edge(john, mary, rel='friends', weight=20)
            :param parent:  edge start node (rgr.Node or node ID)
            :param child:   edge end node (rgr.Node or node ID)
            :param kwargs:  edge properties to initialize
            :return:        rgr.Edge representing the edge you just created.
        """
        new_eid = self.redis.get(self.next_eid_key)
        new_edge = Edge(self, new_eid) 
        self.redis.sadd(self.edges_key, new_eid) 
        for k in kwargs:
            new_edge.prop.__setattr__(k, kwargs[k])
            self._index(new_edge.name, k, kwargs[k]) 
        if type(parent) is Node:
            parent = parent.id
        else:
            parent = str(parent)
        if type(child) is Node:
            child = child.id
        else:
            child = str(child)
        if not self.redis.sismember(self.nodes_key, parent):
            raise ValueError(parent)
        if not self.redis.sismember(self.nodes_key, child):
            raise ValueError(child)
        self.redis.set('{}:e:{}:in'.format(self.name, new_eid), parent)
        self.redis.set('{}:e:{}:on'.format(self.name, new_eid), child)
#.........这里部分代码省略.........

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class Notes(object):

    def __init__(self):
        # initialize redis here
        redis_args = {
            'host': current_app.config['redis']['host'],
            'password': current_app.config['redis']['password'],
            'port': current_app.config['redis']['port']
        }
        self.redis = StrictRedis(**redis_args)

        self.keys = {
            'red': 'red:{user_id}'.format(user_id=g.user_id),
            'blue': 'blue:{user_id}'.format(user_id=g.user_id),
            'green': 'green:{user_id}'.format(user_id=g.user_id)
        }
        self.check_keys = {
            'red': 'red:{user_id}:lcase'.format(user_id=g.user_id),
            'blue': 'blue:{user_id}:lcase'.format(user_id=g.user_id),
            'green': 'green:{user_id}:lcase'.format(user_id=g.user_id)
        }

        self.todays_note_key = '{user_id}:notes:{date}'.format(
            user_id=g.user_id, date=datetime.combine(
                datetime.now(pytz.timezone('US/Pacific')).date(), time())
            .strftime('%y%m%d'))

        self.previous_notes_key = '{user_id}:notes:*'.format(
            user_id=g.user_id)

    def get_notes_count(self):
        with self.redis.pipeline() as pipe:
            pipe.scard(self.keys['red'])
            pipe.scard(self.keys['blue'])
            pipe.scard(self.keys['green'])
            results = pipe.execute()
        return {'red': results[0], 'blue': results[1], 'green': results[2]}

    def get_color(self, type):
        if type in self.keys:
            notes = self.redis.smembers(self.keys[type])
            return notes
        else:
            return []

    def add_notes(self, notes):
        with self.redis.pipeline() as pipe:
            for type, note_list in notes.iteritems():
                for note in note_list:
                    pipe.sadd(self.keys[type], note)
                    pipe.sadd(self.check_keys[type], note.lower())
            pipe.execute()

    def validate_notes(self, notes):
        checked_notes = []
        with self.redis.pipeline() as pipe:
            for type, note_list in notes.iteritems():
                for note in note_list:
                    if type in self.keys:
                        checked_notes.append(note)
                        pipe.sismember(self.check_keys[type], note.lower())
                    else:
                        raise HTTPError(403, 'invalid note type')
            results = pipe.execute()
        err_notes = []
        for i, result in enumerate(results):
            if result == 1:
                err_notes.append(checked_notes[i])
        if err_notes:
            err_message = "Some notes already exist."
            raise HTTPError(403, {'message': err_message, 'notes': err_notes})

    def modify_note(self, type, old_note, new_note):
        if type in self.keys:
            if not self.redis.sismember(self.keys[type], new_note):
                self.delete_note(type, old_note)
                self.add_note(type, new_note)
            else:
                return {'result': 'error', 'err': 'message already in key'}
        else:
            raise HTTPError(403, 'invalid note type')

    def delete_note(self, type, note):
        if type in self.keys:
            self.redis.srem(self.keys[type], note)
            self.redis.srem(self.check_keys[type], note.lower())
            return {'result': 'success'}
        else:
            raise HTTPError(403, 'invalid note type')

    def get_todays_note(self, type=None):
        if not self.redis.exists(self.todays_note_key):
            if not type:
                return None
            note = self.redis.spop(self.keys[type])
            if note:
                note_value = {
                    'note': note,
                    'type': type
                }
#.........这里部分代码省略.........

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class ProxyFetchSpider(Spider):
    name = 'proxy_fetch'
    loop_delay = 10
    protect_sec = 180
    
    def __init__(self, mode='prod', *args, **kwargs):
        if mode == 'prod':            
            LOCAL_CONFIG_YAML = './hq-proxies.yml'
        elif mode == 'test':
            LOCAL_CONFIG_YAML = './hq-proxies.test.yml'
        with open(LOCAL_CONFIG_YAML, 'r', encoding='utf-8') as f:
            LOCAL_CONFIG = yaml.load(f)
        
        self.redis_db = StrictRedis(
            host=LOCAL_CONFIG['REDIS_HOST'], 
            port=LOCAL_CONFIG['REDIS_PORT'], 
            password=LOCAL_CONFIG['REDIS_PASSWORD'],
            db=LOCAL_CONFIG['REDIS_DB']
        )
        self.PROXY_COUNT = LOCAL_CONFIG['PROXY_COUNT']
        self.PROXY_SET = LOCAL_CONFIG['PROXY_SET']

        self.validator_pool = set([])
        for validator in LOCAL_CONFIG['PROXY_VALIDATORS']:
            self.validator_pool.add((validator['url'], validator['startstring']))
        
        self.vendors = LOCAL_CONFIG['PROXY_VENDORS']
    
    def start_requests(self):
        for vendor in self.vendors:
            logger.debug(vendor)
            callback = getattr(self, vendor['parser'])
            yield Request(url=vendor['url'], callback=callback)
    
    def checkin(self, response):
        res = response.body_as_unicode()
        if 'startstring' in response.meta and res.startswith(response.meta['startstring']):
            proxy = response.meta['proxy']
            self.redis_db.sadd(self.PROXY_SET, proxy)
            logger.info('可用代理+1  %s' % proxy)
            yield None
        else:
            proxy = response.url if 'proxy' not in response.meta else response.meta['proxy']
            logger.info('无效代理  %s' % proxy)
            yield None
    
    def parse_xici(self, response):
        ''' 
        @url http://www.xicidaili.com/nn/
        '''
        logger.info('解析http://www.xicidaili.com/nn/')
        succ = 0
        fail = 0
        count = 0
        for tr in response.css('#ip_list tr'):
            td_list = tr.css('td::text')
            if len(td_list) < 3:
                continue
            ipaddr = td_list[0].extract()
            port = td_list[1].extract()
            proto = td_list[5].extract()
            latency = tr.css('div.bar::attr(title)').extract_first()
            latency = re.match('(\d+\.\d+)秒', latency).group(1)
            proxy = '%s://%s:%s' % (proto, ipaddr, port)
            proxies = {proto: '%s:%s' % (ipaddr, port)}
            if float(latency) > 3:
                logger.info('丢弃慢速代理: %s 延迟%s秒' % (proxy, latency))
                continue
            logger.info('验证: %s' % proxy)
            if not self.redis_db.sismember(self.PROXY_SET, proxy):
                vaurl, vastart = random.choice(list(self.validator_pool))
                yield Request(url=vaurl, meta={'proxy': proxy, 'startstring': vastart}, callback=self.checkin, dont_filter=True)
            else:
                logger.info('该代理已收录..')
    
    def parse_66ip(self, response):
        ''' 
        @url http://www.66ip.cn/nmtq.php?getnum=100&isp=0&anonymoustype=3&start=&ports=&export=&ipaddress=&area=1&proxytype=0&api=66ip
        '''
        logger.info('开始爬取66ip')
        if 'proxy' in response.meta:
            logger.info('=>使用代理%s' % response.meta['proxy'])
        res = response.body_as_unicode()
        for addr in re.findall('\d+\.\d+\.\d+\.\d+\:\d+', res):
            proxy = 'http://' + addr
            print(proxy)
            logger.info('验证: %s' % proxy)
            if not self.redis_db.sismember(self.PROXY_SET, proxy):
                vaurl, vastart = random.choice(list(self.validator_pool))
                yield Request(url=vaurl, meta={'proxy': proxy, 'startstring': vastart}, callback=self.checkin, dont_filter=True)
            else:
                logger.info('该代理已收录..')
    
    def parse_ip181(self, response):
        ''' 
        @url http://www.ip181.com/
        '''
        logger.info('开始爬取ip181')
        if 'proxy' in response.meta:
            logger.info('=>使用代理%s' % response.meta['proxy'])
#.........这里部分代码省略.........

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class DpParser(object):
    def __init__(self, config):
        self.config = config
        if 'db_host' in self.config:
            host = self.config['db_host']
        else:
            host = 'localhost'
        self.redis = StrictRedis(host=host, port=6379, db=0)

    def scrape_all(self, url, timestamp, html):
        category, tag, id_ = self.strip_url(url)
        soup = BeautifulSoup(html)

        firsttime = not self.redis.sismember('daftpunk:properties', id_)

        for attr_name in dir(self):
            attr = getattr(self, attr_name)
            try:
                if firsttime and hasattr(attr, '__scrape_once__'):
                    attr(id_, timestamp, soup)
                elif hasattr(attr, '__scrape_update__'):
                    attr(id_, timestamp, soup)
            except Exception as e:
                logging.error("Encountered error parsing %s: %s" % (id_, e))
                logging.error(traceback.format_exc())
                continue

        # Send the rest of message contents to redis
        self.redis.sadd('daftpunk:properties', id_)
        self.redis.rpush('daftpunk:%s:timestamps' % id_, timestamp)
        self.redis.set('daftpunk:%s:url' % id_, url)
        self.redis.set('daftpunk:%s:html' % id_, html)
        self.redis.set('daftpunk:%s:tag' % id_, tag)
        self.redis.set('daftpunk:%s:category' % id_, category)

        return id_

    @scrape_update
    def pricing(self, id_, timestamp, soup):
        price = soup.find(id="smi-price-string")
        if price:
            currency = price.string[0]
            value = price.string[1:].replace(',', '')
            pricing = float(value.split(' ')[0])
            if value.split(' ')[1] == 'Weekly':
                pricing = pricing * 4
            self.redis.zadd('daftpunk:%s:price' % id_, pricing, timestamp)
            self.redis.set('daftpunk:%s:currency' % id_, currency)
            self.redis.set('daftpunk:%s:current_price' % id_, pricing)

    @scrape_once
    def ber_rating(self, id_, timestamp, soup):
        ber = soup.find(**{'class':"ber-icon"})
        ber_number = ber['id'] if ber else ''
        ber_rating = BER_RATINGS.index(ber_number)

        self.redis.set('daftpunk:%s:ber' % id_, ber_rating)

    @scrape_once
    def phone_numbers(self, id_, timestamp, soup):
        phones = set()
        phone_class = soup.find(**{'class':"phone1"})
        if phone_class:
            phone_strs = phone_class.text.split()
            phone_strs = [re_sub('[+()]', '', z) for z in phone_strs]
            for i in reversed(range(len(phone_strs))):
                if not phone_strs[i].isdigit():
                    phones.add('-'.join(phone_strs[i+1:]))
                    phone_strs = phone_strs[:i]

            self.redis.sadd('daftpunk:%s:phone_numbers' % id_, *phones)

    @scrape_once
    def address(self, id_, timestamp, soup):
        address = soup.find(id="address_box").h1.text

        self.redis.set('daftpunk:%s:address' % id_, address)

    @staticmethod
    def get_header_texts(soup):
        headers = soup.find_all(**{'class':'header_text'})
        header_texts = [h.text for h in headers]
        return header_texts

    @scrape_update
    def rooms(self, id_, timestamp, soup):
        headers = self.get_header_texts(soup)

        bedrooms = [h for h in headers if 'bed' in h.lower()]
        bathrooms = [h for h in headers if 'bath' in h.lower()]

        if bedrooms:
            self.redis.set('daftpunk:%s:bedrooms' % id_, bedrooms[0])
        if bathrooms:
            self.redis.set('daftpunk:%s:bathrooms' % id_, bathrooms[0])

    @scrape_once
    def geocode(self, id_, timestamp, soup):
        if not self.is_geocoded(id_):
            address = self.redis.get('daftpunk:%s:address' % id_)
#.........这里部分代码省略.........