本文整理汇总了Python中redis.StrictRedis.sismember方法的典型用法代码示例。如果您正苦于以下问题:Python StrictRedis.sismember方法的具体用法?Python StrictRedis.sismember怎么用?Python StrictRedis.sismember使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类redis.StrictRedis
的用法示例。
在下文中一共展示了StrictRedis.sismember方法的9个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: add_tweets_to_redis
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
def add_tweets_to_redis(tweet_file):
"""
'DVD: FBI WARNING Me: oh boy here we go DVD: The board advises you
to have lots of fun watching this Hollywood movie Me: Ah.. It's a
nice one'
"""
redis_client = StrictRedis(host='localhost', port=6379, db=0)
with open(tweet_file, 'r') as tweets:
for line in tweets:
# again, dealing with weird error here
try:
tweet = line.strip().split('|', 2)[2]
# need to investigate whether one-by-one inserting
# or building a list of tweets and doing a single insert
# would be more efficient
if not redis_client.sismember('tweets', tweet):
result = redis_client.sadd('tweets', tweet)
if not result:
print('error occurred adding tweet: "{}" to redis'
.format(tweet))
else:
# found all new tweets
break
except IndexError:
continue
redis_client.save()
示例2: check_auth
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
def check_auth(login, password, required_scope):
"""This function is called to check if a login, password, and scope combination is valid. """
# Hash the login, the password is ignored
login_hash = calc_hash(login)
cache = StrictRedis(db=config.tokens_cache_redis_db) # TODO: Should be at application scope instead of request scope
user = cache.get(login_hash) # lookup our person
#print('check_auth: login: ' + login + ' scope: '+ required_scope + ', user: ' + str(user))
if (user is None): # if the user has a valid token
return False
elif (required_scope == 'general'):
return True
elif (cache.sismember(required_scope, user) == True):
return True # and the user has the required scope
else:
return False # but the user does not have the required scope
示例3: gdelete
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
def gdelete(request):
groupname = request.POST.get('name', 0);
rd = StrictRedis()
pref = settings.MY_PREFIX
prefg = pref+":"+groupname
user = str(request.user)
print "received request for deleting",groupname,"from",user
ismember = rd.sismember(pref+":groups",groupname)
if not ismember:
return JsonResponse({'done':False,'reason':'No such group name'});
# now check whether the requesting user is the one who created the group
d = rd.hgetall(prefg+":hash")
if d['owner'] != user:
return JsonResponse({'done':False, 'reason':'Only group owner can delete the group'})
rd.srem(pref+":groups", groupname)
rd.delete(prefg+":hash", pref+":"+groupname)
rd.delete(pref+":"+groupname)
redis_publisher = RedisPublisher(facility = pref, broadcast = True)
redis_publisher.publish_message(RedisMessage(json.dumps({"type":"group_delete", "name":groupname})))
return JsonResponse({'done':True})
示例4: Redis
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class Redis(Backend):
"""Default backend for MimicDB. Initiated with identical parameters
as ``redis.StrictRedis``.
:param args \*args, \**kwargs: StrictRedis.__init__() parameters
.. code-block:: python
from mimicdb.backends.default import Redis
redis = Redis(host='localhost', port=6379, db=0)
"""
def __init__(self, *args, **kwargs):
self._redis = StrictRedis(*args, **kwargs)
def keys(self, pattern='*'):
return self._redis.keys(pattern)
def delete(self, *names):
return self._redis.delete(*names)
def sadd(self, name, *values):
return self._redis.sadd(name, *values)
def srem(self, name, *values):
return self._redis.srem(name, *values)
def sismember(self, name, value):
return self._redis.sismember(name, value)
def smembers(self, name):
return self._redis.smembers(name)
def hmset(self, name, mapping):
return self._redis.hmset(name, mapping)
def hgetall(self, name):
return self._redis.hgetall(name)
示例5: MediaWiki
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
#.........这里部分代码省略.........
params = {'action': "sitematrix"}
data = self.query(params)
blacklist_wikis = ['loginwiki']
flaggedrevs_wikis = self._flaggedrevs_wikis()
wikis = {}
for key, val in data['sitematrix'].items():
if key == 'count':
continue
if 'code' in val:
for site in val['site']:
wikis[site['dbname']] = self._create_wiki(
site, val['code'], val['name'],
flaggedrevs_wikis, blacklist_wikis)
else:
for site in val:
wikis[site['dbname']] = self._create_wiki(
site, '', '', flaggedrevs_wikis, blacklist_wikis)
return wikis
def _create_wiki(self, site, langcode, langname, flaggedrevs_wikis,
blacklist_wikis):
wiki = {
'lang': langcode,
'langname': langname,
'url': site['url'].replace("http://", "https://"),
'dbname': site['dbname'],
'group': site['code']
}
if wiki['group'] == 'wiki':
wiki['group'] = 'wikipedia'
inactive_codes = ['closed', 'private', 'fishbowl']
if any([key in site for key in inactive_codes]):
wiki['closed'] = True
if site['dbname'] in blacklist_wikis:
wiki['closed'] = True
if site['dbname'] in flaggedrevs_wikis:
wiki['flaggedrevs'] = True
return wiki
def _flaggedrevs_wikis(self):
url = "https://noc.wikimedia.org/conf/flaggedrevs.dblist"
response = requests.get(url, headers=self.headers)
response.raise_for_status()
return response.text.splitlines()
def ores_context_exists(self, dbname, model='reverted', cached=True):
"""Checks if ORES context for a wiki exists"""
key = config.redis_prefix + 'ores' + model
if not self.redis.exists(key) or not cached:
self._ores_contexts(model)
return self.redis.sismember(key, dbname)
def _ores_contexts(self, model):
"""Fill cache"""
pipe = self.redis.pipeline()
key = config.redis_prefix + 'ores' + model
pipe.delete(key)
contexts = requests.get(self.ores_url, headers=self.headers)
contexts.raise_for_status()
contexts = contexts.json()['contexts']
for context in contexts:
models = requests.get('{}/{}/'.format(self.ores_url, context),
headers=self.headers)
models.raise_for_status()
models = models.json()['models']
if model in models:
pipe.sadd(key, context)
pipe.expire(key, 172800) # 2 days exp.
pipe.execute()
def ores_scores(self, dbname, revids, model='reverted'):
"""Get ORES scores for revision ids"""
url = '{}/{}/{}/'.format(self.ores_url, dbname, model)
revids = '|'.join([str(id) for id in revids])
params = {'revids': revids}
response = requests.get(url, params=params, headers=self.headers)
if response.status_code != requests.codes.ok:
raise Exception('ORES error code {} for {} with params {}'.format(
response.status_code, dbname, str(params)), response.text)
return response.json()
def was_reverted(self, revision):
"""
Checks if a revision was reverted
:param revision: a revision dict containing ‘revid’ and ‘pageid’
"""
session = RevertsSession(self.api_url, user_agent=self.user_agent)
return reverts.check_rev(session, revision)
示例6: Graph
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class Graph(object):
"""
Main interface for adding/removing/looking up
graph elements.
basic usage:
you'll add and remove elements via:
-add_node()/add_edge()
-del_node()/del_edge()
you can also look up nodes by property:
get_nodes()/get_edges()
you can query by property with regex:
find_nodes()/find_edges()
all of these methods will return a list of
Node() or Edge() objects. From these, you are
able to look at incoming or ourgoing edges,
parent and child nodes, and individually access
and modify their properties.
Usage::
>>> from rgr import Graph
>>> g = Graph()
>>> g2 = Graph('mygraph')
:param name: default -> 'rgr'; the namespace for your graph within redis.
"""
#TODO: better errors
def __init__(self, name='rgr'):
self.redis = Redis()
self.name = name
self.next_nid_key = self.name + ':next_nid' #scalar
self.next_eid_key = self.name + ':next_eid' #scalar
self.nodes_key = self.name + ':nodes' #set
self.edges_key = self.name + ':edges' #set
if not self.redis.exists(self.next_nid_key):
self.redis.set(self.next_nid_key, 0)
if not self.redis.exists(self.next_eid_key):
self.redis.set(self.next_eid_key, 0)
def add_node(self, **kwargs):
"""
Add a node to the graph.
Usage::
>>> from rgr import Graph
>>> g = rgr.Graph()
>>> n = g.add_node() #blank node
>>> n2 = g.add_node(name='john', type='person')
:param kwargs: node properties to initialize
:return: rgr.Node representing the node you just created.
"""
new_nid = self.redis.get(self.next_nid_key)
new_node = Node(self, new_nid)
self.redis.sadd(self.nodes_key, new_nid)
for k in kwargs:
new_node.prop.__setattr__(k, kwargs[k])
self._index(new_node.name, k, kwargs[k])
self.redis.incr(self.next_nid_key)
return new_node
def add_edge(self, parent, child, **kwargs):
"""
Add an edge between two nodes.
Usage::
>>> from rgr import Graph
>>> g = Graph()
>>> john = g.add_node(name='john')
>>> mary = g.add_node(name='mary')
>>> e = g.add_edge(john, mary, rel='friends', weight=20)
:param parent: edge start node (rgr.Node or node ID)
:param child: edge end node (rgr.Node or node ID)
:param kwargs: edge properties to initialize
:return: rgr.Edge representing the edge you just created.
"""
new_eid = self.redis.get(self.next_eid_key)
new_edge = Edge(self, new_eid)
self.redis.sadd(self.edges_key, new_eid)
for k in kwargs:
new_edge.prop.__setattr__(k, kwargs[k])
self._index(new_edge.name, k, kwargs[k])
if type(parent) is Node:
parent = parent.id
else:
parent = str(parent)
if type(child) is Node:
child = child.id
else:
child = str(child)
if not self.redis.sismember(self.nodes_key, parent):
raise ValueError(parent)
if not self.redis.sismember(self.nodes_key, child):
raise ValueError(child)
self.redis.set('{}:e:{}:in'.format(self.name, new_eid), parent)
self.redis.set('{}:e:{}:on'.format(self.name, new_eid), child)
#.........这里部分代码省略.........
示例7: Notes
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class Notes(object):
def __init__(self):
# initialize redis here
redis_args = {
'host': current_app.config['redis']['host'],
'password': current_app.config['redis']['password'],
'port': current_app.config['redis']['port']
}
self.redis = StrictRedis(**redis_args)
self.keys = {
'red': 'red:{user_id}'.format(user_id=g.user_id),
'blue': 'blue:{user_id}'.format(user_id=g.user_id),
'green': 'green:{user_id}'.format(user_id=g.user_id)
}
self.check_keys = {
'red': 'red:{user_id}:lcase'.format(user_id=g.user_id),
'blue': 'blue:{user_id}:lcase'.format(user_id=g.user_id),
'green': 'green:{user_id}:lcase'.format(user_id=g.user_id)
}
self.todays_note_key = '{user_id}:notes:{date}'.format(
user_id=g.user_id, date=datetime.combine(
datetime.now(pytz.timezone('US/Pacific')).date(), time())
.strftime('%y%m%d'))
self.previous_notes_key = '{user_id}:notes:*'.format(
user_id=g.user_id)
def get_notes_count(self):
with self.redis.pipeline() as pipe:
pipe.scard(self.keys['red'])
pipe.scard(self.keys['blue'])
pipe.scard(self.keys['green'])
results = pipe.execute()
return {'red': results[0], 'blue': results[1], 'green': results[2]}
def get_color(self, type):
if type in self.keys:
notes = self.redis.smembers(self.keys[type])
return notes
else:
return []
def add_notes(self, notes):
with self.redis.pipeline() as pipe:
for type, note_list in notes.iteritems():
for note in note_list:
pipe.sadd(self.keys[type], note)
pipe.sadd(self.check_keys[type], note.lower())
pipe.execute()
def validate_notes(self, notes):
checked_notes = []
with self.redis.pipeline() as pipe:
for type, note_list in notes.iteritems():
for note in note_list:
if type in self.keys:
checked_notes.append(note)
pipe.sismember(self.check_keys[type], note.lower())
else:
raise HTTPError(403, 'invalid note type')
results = pipe.execute()
err_notes = []
for i, result in enumerate(results):
if result == 1:
err_notes.append(checked_notes[i])
if err_notes:
err_message = "Some notes already exist."
raise HTTPError(403, {'message': err_message, 'notes': err_notes})
def modify_note(self, type, old_note, new_note):
if type in self.keys:
if not self.redis.sismember(self.keys[type], new_note):
self.delete_note(type, old_note)
self.add_note(type, new_note)
else:
return {'result': 'error', 'err': 'message already in key'}
else:
raise HTTPError(403, 'invalid note type')
def delete_note(self, type, note):
if type in self.keys:
self.redis.srem(self.keys[type], note)
self.redis.srem(self.check_keys[type], note.lower())
return {'result': 'success'}
else:
raise HTTPError(403, 'invalid note type')
def get_todays_note(self, type=None):
if not self.redis.exists(self.todays_note_key):
if not type:
return None
note = self.redis.spop(self.keys[type])
if note:
note_value = {
'note': note,
'type': type
}
#.........这里部分代码省略.........
示例8: ProxyFetchSpider
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class ProxyFetchSpider(Spider):
name = 'proxy_fetch'
loop_delay = 10
protect_sec = 180
def __init__(self, mode='prod', *args, **kwargs):
if mode == 'prod':
LOCAL_CONFIG_YAML = './hq-proxies.yml'
elif mode == 'test':
LOCAL_CONFIG_YAML = './hq-proxies.test.yml'
with open(LOCAL_CONFIG_YAML, 'r', encoding='utf-8') as f:
LOCAL_CONFIG = yaml.load(f)
self.redis_db = StrictRedis(
host=LOCAL_CONFIG['REDIS_HOST'],
port=LOCAL_CONFIG['REDIS_PORT'],
password=LOCAL_CONFIG['REDIS_PASSWORD'],
db=LOCAL_CONFIG['REDIS_DB']
)
self.PROXY_COUNT = LOCAL_CONFIG['PROXY_COUNT']
self.PROXY_SET = LOCAL_CONFIG['PROXY_SET']
self.validator_pool = set([])
for validator in LOCAL_CONFIG['PROXY_VALIDATORS']:
self.validator_pool.add((validator['url'], validator['startstring']))
self.vendors = LOCAL_CONFIG['PROXY_VENDORS']
def start_requests(self):
for vendor in self.vendors:
logger.debug(vendor)
callback = getattr(self, vendor['parser'])
yield Request(url=vendor['url'], callback=callback)
def checkin(self, response):
res = response.body_as_unicode()
if 'startstring' in response.meta and res.startswith(response.meta['startstring']):
proxy = response.meta['proxy']
self.redis_db.sadd(self.PROXY_SET, proxy)
logger.info('可用代理+1 %s' % proxy)
yield None
else:
proxy = response.url if 'proxy' not in response.meta else response.meta['proxy']
logger.info('无效代理 %s' % proxy)
yield None
def parse_xici(self, response):
'''
@url http://www.xicidaili.com/nn/
'''
logger.info('解析http://www.xicidaili.com/nn/')
succ = 0
fail = 0
count = 0
for tr in response.css('#ip_list tr'):
td_list = tr.css('td::text')
if len(td_list) < 3:
continue
ipaddr = td_list[0].extract()
port = td_list[1].extract()
proto = td_list[5].extract()
latency = tr.css('div.bar::attr(title)').extract_first()
latency = re.match('(\d+\.\d+)秒', latency).group(1)
proxy = '%s://%s:%s' % (proto, ipaddr, port)
proxies = {proto: '%s:%s' % (ipaddr, port)}
if float(latency) > 3:
logger.info('丢弃慢速代理: %s 延迟%s秒' % (proxy, latency))
continue
logger.info('验证: %s' % proxy)
if not self.redis_db.sismember(self.PROXY_SET, proxy):
vaurl, vastart = random.choice(list(self.validator_pool))
yield Request(url=vaurl, meta={'proxy': proxy, 'startstring': vastart}, callback=self.checkin, dont_filter=True)
else:
logger.info('该代理已收录..')
def parse_66ip(self, response):
'''
@url http://www.66ip.cn/nmtq.php?getnum=100&isp=0&anonymoustype=3&start=&ports=&export=&ipaddress=&area=1&proxytype=0&api=66ip
'''
logger.info('开始爬取66ip')
if 'proxy' in response.meta:
logger.info('=>使用代理%s' % response.meta['proxy'])
res = response.body_as_unicode()
for addr in re.findall('\d+\.\d+\.\d+\.\d+\:\d+', res):
proxy = 'http://' + addr
print(proxy)
logger.info('验证: %s' % proxy)
if not self.redis_db.sismember(self.PROXY_SET, proxy):
vaurl, vastart = random.choice(list(self.validator_pool))
yield Request(url=vaurl, meta={'proxy': proxy, 'startstring': vastart}, callback=self.checkin, dont_filter=True)
else:
logger.info('该代理已收录..')
def parse_ip181(self, response):
'''
@url http://www.ip181.com/
'''
logger.info('开始爬取ip181')
if 'proxy' in response.meta:
logger.info('=>使用代理%s' % response.meta['proxy'])
#.........这里部分代码省略.........
示例9: DpParser
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import sismember [as 别名]
class DpParser(object):
def __init__(self, config):
self.config = config
if 'db_host' in self.config:
host = self.config['db_host']
else:
host = 'localhost'
self.redis = StrictRedis(host=host, port=6379, db=0)
def scrape_all(self, url, timestamp, html):
category, tag, id_ = self.strip_url(url)
soup = BeautifulSoup(html)
firsttime = not self.redis.sismember('daftpunk:properties', id_)
for attr_name in dir(self):
attr = getattr(self, attr_name)
try:
if firsttime and hasattr(attr, '__scrape_once__'):
attr(id_, timestamp, soup)
elif hasattr(attr, '__scrape_update__'):
attr(id_, timestamp, soup)
except Exception as e:
logging.error("Encountered error parsing %s: %s" % (id_, e))
logging.error(traceback.format_exc())
continue
# Send the rest of message contents to redis
self.redis.sadd('daftpunk:properties', id_)
self.redis.rpush('daftpunk:%s:timestamps' % id_, timestamp)
self.redis.set('daftpunk:%s:url' % id_, url)
self.redis.set('daftpunk:%s:html' % id_, html)
self.redis.set('daftpunk:%s:tag' % id_, tag)
self.redis.set('daftpunk:%s:category' % id_, category)
return id_
@scrape_update
def pricing(self, id_, timestamp, soup):
price = soup.find(id="smi-price-string")
if price:
currency = price.string[0]
value = price.string[1:].replace(',', '')
pricing = float(value.split(' ')[0])
if value.split(' ')[1] == 'Weekly':
pricing = pricing * 4
self.redis.zadd('daftpunk:%s:price' % id_, pricing, timestamp)
self.redis.set('daftpunk:%s:currency' % id_, currency)
self.redis.set('daftpunk:%s:current_price' % id_, pricing)
@scrape_once
def ber_rating(self, id_, timestamp, soup):
ber = soup.find(**{'class':"ber-icon"})
ber_number = ber['id'] if ber else ''
ber_rating = BER_RATINGS.index(ber_number)
self.redis.set('daftpunk:%s:ber' % id_, ber_rating)
@scrape_once
def phone_numbers(self, id_, timestamp, soup):
phones = set()
phone_class = soup.find(**{'class':"phone1"})
if phone_class:
phone_strs = phone_class.text.split()
phone_strs = [re_sub('[+()]', '', z) for z in phone_strs]
for i in reversed(range(len(phone_strs))):
if not phone_strs[i].isdigit():
phones.add('-'.join(phone_strs[i+1:]))
phone_strs = phone_strs[:i]
self.redis.sadd('daftpunk:%s:phone_numbers' % id_, *phones)
@scrape_once
def address(self, id_, timestamp, soup):
address = soup.find(id="address_box").h1.text
self.redis.set('daftpunk:%s:address' % id_, address)
@staticmethod
def get_header_texts(soup):
headers = soup.find_all(**{'class':'header_text'})
header_texts = [h.text for h in headers]
return header_texts
@scrape_update
def rooms(self, id_, timestamp, soup):
headers = self.get_header_texts(soup)
bedrooms = [h for h in headers if 'bed' in h.lower()]
bathrooms = [h for h in headers if 'bath' in h.lower()]
if bedrooms:
self.redis.set('daftpunk:%s:bedrooms' % id_, bedrooms[0])
if bathrooms:
self.redis.set('daftpunk:%s:bathrooms' % id_, bathrooms[0])
@scrape_once
def geocode(self, id_, timestamp, soup):
if not self.is_geocoded(id_):
address = self.redis.get('daftpunk:%s:address' % id_)
#.........这里部分代码省略.........