Python StrictRedis.rpop方法代码示例

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import rpop [as 别名]
class Worker(object):
    def __init__(self, queue_name=None, from_queue=None, host='127.0.0.1', port=6379, db=0, prefix='easy_jobs:queue:'):
        self.queue_name = queue_name
        self.from_queue = from_queue
        self.outputs = ['results']

        if from_queue:
            self.connection = self.from_queue.connection
            self.key = self.from_queue.key
        else:
            self.key = '%s%s' % (prefix, queue_name)
            self.connection = StrictRedis(host=host, port=port, db=db)

    def start(self, once=False, threaded=False):
        jobs = self.connection.llen(self.key)

        while jobs > 0:
            pickled = self.connection.rpop(self.key)

            if pickled:
                un_pickled = loads(pickled)
                instance = un_pickled['foo']
                args = un_pickled['args']
                kwargs = un_pickled['kwargs']
                p = Process(target=instance, args=args, kwargs=kwargs)
                p.start()

                if not threaded:
                    p.join()
            jobs = self.connection.llen(self.key) if not once else -1

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import rpop [as 别名]
class JSONRedisBackend(BaseBackend):

    def __init__(self):
        self.redis = StrictRedis()

    def serialize(self, value):
        return json.dumps(value)

    def deserialize(self, value):
        return value and json.loads(value) or None

    def get(self, key):
        value = self.redis.get(key)
        result = self.deserialize(value)
        return result

    def set(self, key, value):
        product = self.serialize(value)
        return self.redis.set(key, product)

    def rpush(self, key, value):
        product = self.serialize(value)
        return self.redis.rpush(key, product)

    def lpush(self, key, value):
        product = self.serialize(value)
        return self.redis.lpush(key, product)

    def lpop(self, key):
        value = self.redis.lpop(key)
        result = self.deserialize(value)
        return result

    def llen(self, key):
        return self.redis.llen(key)

    def rpop(self, key):
        value = self.redis.rpop(key)
        result = self.deserialize(value)
        return result

    def report_steps(self, name, consumers, producers):
        pipeline = self.redis.pipeline()
        producers_key = ':'.join([name, 'producers'])
        consumers_key = ':'.join([name, 'consumers'])

        for consumer in consumers:
            pipeline.sadd(consumers_key, consumer)

        for producer in producers:
            pipeline.sadd(producers_key, producer)

        pipeline.smembers(consumers_key)
        pipeline.smembers(producers_key)

        result = pipeline.execute()
        all_consumers = result[-2]
        all_producers = result[-1]

        return all_consumers, all_producers

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import rpop [as 别名]
class VisualDebugger:

    def __init__(self):
        self.available_buckets = config["visual_debugger"]["available_buckets"]
        self.bucket_generator = itertools.cycle(self.available_buckets)

        self.redis_client = StrictRedis(**config["redis"])
        self.clear_image_data()

    def store_image_data(self, image_data, image_shape, bucket="debug"):
        self.redis_client.lpush(f"{config['visual_debugger']['redis_key_prefix']}:{bucket}:SHAPE", pickle.dumps(image_shape))
        self.redis_client.lpush(f"{config['visual_debugger']['redis_key_prefix']}:{bucket}", image_data.tobytes())

    def retrieve_image_data(self):
        bucket = next(self.bucket_generator)
        bucket_key = f"{config['visual_debugger']['redis_key_prefix']}:{bucket}"

        response = self.redis_client.rpop(bucket_key)

        if response is not None:
            bucket = bucket_key.split(":")[-1]

            image_shape = self.redis_client.rpop(f"{config['visual_debugger']['redis_key_prefix']}:{bucket}:SHAPE")
            image_shape = pickle.loads(image_shape)

            image_data = np.fromstring(response, dtype="uint8").reshape(image_shape)

            return bucket, image_data

        return None

    def save_image_data(self, bucket, image_data):
        if bucket in self.available_buckets:
            if image_data.dtype == "bool" or (image_data.dtype == "uint8" and 1 in np.unique(image_data)):
                image_data = image_data.astype("uint8") * 255

            skimage.io.imsave(f"{bucket}.png", image_data)

    def clear_image_data(self):
        visual_debugger_keys = self.redis_client.keys(f"{config['visual_debugger']['redis_key_prefix']}*")

        for key in visual_debugger_keys:
            self.redis_client.delete(key.decode("utf-8"))

    def get_bucket_queue_length(self, bucket):
        return self.redis_client.llen(f"{config['visual_debugger']['redis_key_prefix']}:{bucket}")

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import rpop [as 别名]
 def test_filled_list_contents(self) -> None:
     """
     Test if there are entries in the list (does the preparation work as expected?)
     """
     fill_items = (
         'item_1',
         'item_2',
         'item_three',
     )
     redis_filler = StrictRedis(connection_pool=self.__pool)
     for item in fill_items:
         redis_filler.lpush(self.__config['queue'], item)
     rqa = RedisQueueAccess(self.__config)
     redis_receiver = StrictRedis(connection_pool=rqa.create_redis_connection_pool())
     for item in fill_items:
         data = redis_receiver.rpop(rqa.queue)
         self.assertEqual(bytes(item, encoding='UTF-8'), data)
     self.assertIsNone(redis_receiver.rpop(rqa.queue))

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import rpop [as 别名]
def _subscribe(channel):
    r = Redis(host="localhost", port=6379, db=0)

    while True:
        try:
            msg = r.rpop(channel)
        except Exception, e:
            msg = e
        if msg:
            print("channel: {} \nmessage: {}".format(channel, msg))

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import rpop [as 别名]
class JSONRedisBackend(BaseBackend):
    def initialize(self):
        conf = env.get_uri("LINEUP_REDIS_URI")

        self.redis = StrictRedis(
            db=conf.username or 0,
            host=conf.host,
            port=conf.port,
            # using `path` as password to support the URI like:
            # redis://[email protected]:port/veryverylongpasswordhash
            password=conf.path,
        )

    def serialize(self, value):
        return json.dumps(value, default=bytes)

    def deserialize(self, value):
        return value and json.loads(value) or None

    # read operations
    @io_operation
    def get(self, key):
        value = self.redis.get(key)
        result = self.deserialize(value)
        return result

    @io_operation
    def lpop(self, key):
        value = self.redis.lpop(key)
        result = self.deserialize(value)
        return result

    @io_operation
    def llen(self, key):
        return self.redis.llen(key)

    @io_operation
    def lrange(self, key, start, stop):
        return map(self.deserialize, self.redis.lrange(key, start, stop))

    @io_operation
    def rpop(self, key):
        value = self.redis.rpop(key)
        result = self.deserialize(value)
        return result

    # Write operations
    @io_operation
    def set(self, key, value):
        product = self.serialize(value)
        return self.redis.set(key, product)

    @io_operation
    def rpush(self, key, value):
        product = self.serialize(value)
        return self.redis.rpush(key, product)

    @io_operation
    def lpush(self, key, value):
        product = self.serialize(value)
        return self.redis.lpush(key, product)

    # Pipeline operations
    @io_operation
    def report_steps(self, name, consumers, producers):
        pipeline = self.redis.pipeline()
        producers_key = ':'.join([name, 'producers'])
        consumers_key = ':'.join([name, 'consumers'])

        for consumer in consumers:
            pipeline.sadd(consumers_key, consumer)

        for producer in producers:
            pipeline.sadd(producers_key, producer)

        pipeline.smembers(consumers_key)
        pipeline.smembers(producers_key)

        result = pipeline.execute()
        all_consumers = result[-2]
        all_producers = result[-1]

        return all_consumers, all_producers

# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import rpop [as 别名]
class Screenshot(QWebView):
    
    def __init__(self):
        self.app = QApplication(sys.argv)
        QWebView.__init__(self)
        self._loaded = False
        self.redis_cli = StrictRedis(host='localhost', port=6379, db=0)
        self.loadFinished.connect(self._loadFinished)
        self.replace_sign = 'replace_me'
        self.containers = {}
        self.bad_screenshots = Image.open('/mnt/screenshot/bad/1.png').histogram()

    def _create_script(self, html):
        return 'decoder=document.getElementById("decoder");decoder.innerHTML="%s";document.getElementById("screenshot_container").innerHTML=decoder.innerText;' % cgi.escape(html, quote=True)
    
    def start(self):
        while True:
            keys = self.redis_cli.keys('*_screenshot_requests')                
            for key in keys:
                spider_name = key.split('_')[0]
                if spider_name not in self.containers.keys():
                    self.containers[spider_name] = Container(spider_name)     
                req = self.redis_cli.rpop(key)
                while req:
                    self.process_request(req, spider_name)
                    req = self.redis_cli.rpop(key)
            print 'sleep for 10 seconds'
            time.sleep(10)
            

    def process_request(self, request, spider_name):
        container = self.containers[spider_name]
        request = eval(request)
        question_id = request[0]
        name = request[1]
        data = request[2]
        folder_path = os.path.join('/mnt/screenshots', question_id[:2], question_id)
        if not os.path.exists(folder_path):
            os.makedirs(folder_path)
        if data:
            out_file = os.path.join(folder_path, '%s.png' % name)
            if not os.path.exists(out_file):
                html = container.container.replace(self.replace_sign, data)
                html_file = os.path.join(container.container_path, '%s.html' % name)
                with open(html_file, 'wb') as f:
                    f.write(html.encode('utf8'))
                    f.flush()
                repeater = 5
                while repeater > 0:
                    repeater = repeater - 1
                    result = self.capture('file://%s' % os.path.abspath(html_file), out_file)
                    if result:
                        break
                #log this failure    
                if repearter == 0:
                    redis_cli.sadd('bad_screenshots', question_id)
                    
                os.unlink(html_file)

    def capture(self, html_file, output_file):
        self.load(QUrl(html_file))
        self.wait_load()
        # set to webpage size
        frame = self.page().mainFrame()
        #dump = frame.evaluateJavaScript(QString(self._create_script(html)))
        dump2 = frame.evaluateJavaScript(QString('hiddenAnswers()'))
        qv = frame.evaluateJavaScript(QString('document.getElementById("screenshot_container").scrollHeight;'))
        height_info = qv.toInt() 
        size = frame.contentsSize()
        #ensure width is 700
        size.setWidth(700)
        if height_info[1]:
            size.setHeight(height_info[0])
            
        self.page().setViewportSize(size)
        # render image
        image = QImage(self.page().viewportSize(), QImage.Format_ARGB32)
        painter = QPainter(image)
        frame.render(painter)
        painter.end()
        image.save(output_file)
        if is_blank(output_file) or is_invalid(output_file, self.bad_screenshots):
            print 'failure when capturing %s' % output_file
            return False
        print 'succeed on capturing %s' % output_file    
        return True

    def wait_load(self, delay=0):
        # process app events until page loaded
        while not self._loaded:
            self.app.processEvents()
            time.sleep(delay)
        self._loaded = False

    def _loadFinished(self, result):
        self._loaded = True
        
    def generate_path(self, question_id):
        '''generate path for url'''
        #key = url_fingerprint(url)
#.........这里部分代码省略.........