Python StrictRedis.lrem方法代码示例

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import lrem [as 别名]
class DbRedis(Db):
    def __init__(self):
        super().__init__()
        self.redis = StrictRedis(**dbSettings)

    def generateKey(self):
        return self.redis.incr("id")

    def store(self, key, objSerial):
        self.redis.setex(key, TTL, objSerial)

    def retrieve(self, key):
        return self.redis.get(key)

    def lenList(self, name):
        return self.redis.llen(name)

    def listAppend(self, name, val):
        self.redis.lpush(name, val)

    def listPopLeft(self, name):
        return self.redis.lpop(name)

    def retrieveList(self, name):
        return self.redis.lrange(name, 0, -1)

    def removeFromList(self, name, item):
        self.redis.lrem(name, item, 0)

    def keyExists(self, key):
        return self.redis.exists(key)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import lrem [as 别名]
 def perform_action(self, request, *args, **kwargs):
     r = StrictRedis(settings.REDIS_HOST,
                     settings.REDIS_PORT,
                     settings.REDIS_DB)
     index = kwargs['action_item']
     tmp = uuid.uuid4().hex
     r.lset('users:{0}:action.items'.format(self.request.user.id), index, tmp)
     r.lrem('users:{0}:action.items'.format(self.request.user.id), 0, tmp)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import lrem [as 别名]
class Hipache(object):

    def __init__(self, url):
        host, port, password, database = parse_redis_url(url)

        self.r = StrictRedis(host=host, port=port, db=database, password=password)

    def __iter__(self):
        for i, k in enumerate(self.r.keys()):
            vhost = k.split(":")[1]
            xs = self.r.lrange(k, 0, -1)
            if len(xs) == 2:
                id, url = xs
                yield {"id": id, "host": vhost, "url": url}

    def add(self, id, host, ip, port):
        url = format_url(ip, port)
        vhost = "frontend:{0:s}".format(host)
        if vhost in self.r.keys():
            members = self.r.lrange(vhost, 0, -1)
            if id in members:
                if url not in members:
                    self.r.linsert(vhost, "after", id, url)
            else:
                self.r.rpush(vhost, id)
                self.r.rpush(vhost, url)
        else:
            self.r.rpush(vhost, id)
            self.r.rpush(vhost, url)

    def delete(self, id, host, ip, port):
        vhost = "frontend:{0:s}".format(host)
        if not ip:
            self.r.delete(vhost, id)
        else:
            url = format_url(ip, port)
            self.r.lrem(vhost, 0, url)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import lrem [as 别名]
 def handle(self, *args, **options):
     daemon = BeatmapDaemon()
     daemon.crawler.start_session()
     page = 0
     flag = True
     redis = StrictRedis()
     while flag:
         flag = False
         page += 1
         url = "https://osu.ppy.sh/p/beatmaplist&s=4&r=0&page=%d" % page
         beatmap_ids = set(daemon.crawler.parse_listing_page(daemon.crawler.session.get(url).text))
         for beatmap in Beatmap.objects.filter(id__in=beatmap_ids).all():
             beatmap_ids.remove(beatmap.id)
         for beatmap_id in beatmap_ids:
             if redis.lrem(FETCH_QUEUE_KEY, 0, beatmap_id) == 0L:
                 flag = True
             redis.rpush(FETCH_QUEUE_KEY, beatmap_id)
             self.stderr.write('Enqueued #%d.' % beatmap_id)

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import lrem [as 别名]
class FacebookSpider(Spider):
  name = 'fb_spider'
  count = 0

  directory_links_regex = r'.*/directory/people.*'
  people_links_regex = r'.*\.facebook\.com/[a-zA-Z0-9\.]+$'

  def __init__(self, debug=None, remote=False, *args, **kwargs):
    if debug:
      print 'debug mode on'
      self.allowed_domains = ["localhost:5000"]
      self.start_urls = ["http://localhost:5000/directory/people"]
    else:
      self.allowed_domains = ["facebook.com"]
      self.start_urls = ["https://www.facebook.com/directory/people"]

    if remote:
      self.redis = StrictRedis('54.172.41.190')
    else:
      self.redis = StrictRedis()

  def remove_proxy(self, ip):
    # remove the proxy from the list;
    # terminate proxy;
    # fire up a new proxy
    res = self.redis.lrem('plist', 0, ip)
    if res > 0:
      # only start a new instance if the remove actually worked
      print 'removing proxy', ip
      stop_instance_by_ip(ip)
      print 'starting new proxy...'
      start_instances(1)

  def start_requests(self):
    r = Request(
      url='https://www.facebook.com/directory/people',
      meta={'count': 0}
    )
    return [r]

  def parse(self, response):
    lh = lx.fromstring(response.body)

    # handle potential Captchas here
    if 'Security Check' in response.body:
      # resubmit request & stop processing
      print 'hit a security check...'
      proxy = response.request.meta.get('proxy')
      if proxy:
        print 'attemping to remove proxy', proxy
        self.remove_proxy(proxy)
      yield response.request
      return

    ###########################

    links = lh.xpath('//a')
    for l in links:
      href = l.attrib.get('href')
      if href:
        absolute_href = urljoin(response.url, href)
        if re.match(self.directory_links_regex, absolute_href):
          self.count += 1
          request = Request(absolute_href, callback=self.parse, meta={'count': self.count})
          yield request
        elif re.match(self.people_links_regex, href):
          item = FbnamesItem()
          item['anchor'] = l.text_content().strip()
          item['link'] = absolute_href
          yield item

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import lrem [as 别名]

#.........这里部分代码省略.........
        result = dict()
        for folder in self.__files.get_contentsub(full=False):
            result[folder] = len(self.get_images(folder=folder))
        return result

    def get_all_images(self):
        """suppenkasper needs a list of all images"""
        result = list()
        for folder in self.__files.get_contentsub(full=False):
            result += self.get_images(folder=folder)
        return result

    def get_dict_images(self, folder):
        result = dict()
        if folder in self.__files.get_contentsub(full=False):
            for image in sorted(self.get_images(folder=folder)):
                result[image] = self.locate_image(image)
            return result

    def get_sort_images(self, folder="unsorted", page=0):
        """batch of images to sort"""
        result = dict()
        if folder in self.__files.get_contentsub(full=False):
            for image in sorted(self.get_images(folder=folder))[
                page * self.__redis_opt["sort_slices"] : page * self.__redis_opt["sort_slices"]
                + self.__redis_opt["sort_slices"]
            ]:
                result[image] = self.locate_image(image)
            return result

    def random_image(self, folder="public"):
        """just one of those images"""
        if folder in self.__files.get_contentsub(full=False):
            images = self.get_images(folder)
            if images:
                result = self.locate_image(choice(images))
                return result if result else self.__files.jinja_nullimg()

    def move_image(self, name, target):
        """moves images"""
        folder = self.__dblocate_image(name)
        if folder and target in self.__files.get_contentsub(full=False):
            sourcefile = self.__files.jinja_static_file(name, folder=folder, full=True)
            targetfile = self.__files.path_join(target, name)
            if sourcefile and targetfile:
                rdbsourcefield = "%s:%s" % (self.__redis_opt["image_prefix"], folder)
                rdbtargetfield = "%s:%s" % (self.__redis_opt["image_prefix"], target)
                self.__rdb.rpush(rdbtargetfield, name)
                self.__rdb.lrem(rdbsourcefield, 0, name)
                self.__files.file_rename(sourcefile, targetfile)
                print("<- %s\n-> %s\n[ %s %s ]\n\n" % (sourcefile, targetfile, rdbsourcefield, rdbtargetfield))

    #

    def get_status(self):
        """gets status from redis"""
        result = dict()
        rdbfield = "%s:feed" % (self.__redis_opt["status_prefix"])

        def __readin():
            """reloads db"""
            self.__rdb.delete(rdbfield)
            result = self.__net.url_scrape(self.__redis_opt["status_url"])
            if result:
                self.__rdb.set(rdbfield, result)
                self.__rdb.expire(rdbfield, self.__redis_opt["status_expire"])
                LOGGER.info(
                    "rebuilt redis status cache for %s, again after %i seconds"
                    % (rdbfield, self.__redis_opt["status_expire"])
                )
                return loads(result)

        result = self.__rdb.get(rdbfield)
        return loads(result) if result else __readin()

    #

    def browser_shout(self, channel):
        pubsub = self.__rdb.pubsub()
        pubsub.subscribe(channel)
        for event in pubsub.listen():
            LOGGER.info("shouting to browser channel:%s message:%s" % (channel, event["data"]))
            if event["type"] == "message":
                strdata = "data: %s\r\n\r\n" % (event["data"].replace("\n", "<br />"))
                yield strdata.encode("UTF-8")

    def redis_shout(self, channel, message):
        LOGGER.info("shouting to redis channel:%s message:%s" % (channel, message))
        self.__rdb.publish(channel, message)
        return message

    #

    def next_image(self):
        """shout next random image from public to redis"""
        image = self.random_image(folder="public")
        if image:
            LOGGER.info("shouting next image: %s" % (image))
            self.redis_shout(self.__redis_opt["image_pubsub"], image)
            print("shouting next image: %s" % (image))