Python hotqueue.HotQueue类代码示例

 def test_arguments(self):
     """Test that HotQueue.__init__ accepts arguments correctly, and that
     the Redis key is correctly formed.
     """
     kwargs = {
         'name': "testqueue",
         'serializer': DummySerializer,
         'host': "localhost",
         'port': 6379,
         'db': 0}
     # Instantiate the HotQueue instance:
     self.queue = HotQueue(**kwargs)
     # Ensure that the properties of the instance are as expected:
     self.assertEqual(self.queue.name, kwargs['name'])
     self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
     self.assertEqual(self.queue.serializer, kwargs['serializer'])
     self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host'])
     self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host'])
     self.assertEqual(self.queue._HotQueue__redis.port, kwargs['port'])
     self.assertEqual(self.queue._HotQueue__redis.db, kwargs['db'])
     # Instantiate a HotQueue instance with only the required args:
     self.queue = HotQueue(kwargs['name'])
     # Ensure that the properties of the instance are as expected:
     self.assertEqual(self.queue.name, kwargs['name'])
     self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
     self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle or

def make(name, path, sentinals):
    q = HotQueue(name)
    with open(path) as f:
        for line in f.readlines():
            q.put(line.strip().split('\t'))

    for n in xrange(sentinals):
        q.put(None)

 def unsubscribe_uuid(cls, uuid):
     if True: #TODO: check uuid
         queue = HotQueue("connection_thread_id_queue=" + str(g.connection_thread_id))
         msg = {'cmd': 'unsubscribe', 'params': str(uuid)}
         queue.put(json.dumps(msg))
         return msg
     else:
         return None

 def open(self):
     print "Opening"
     logging.info("opening")
     print "Initializing queue"
     queue = HotQueue("myqueue", host="localhost", port=6379, db=0)
     count = 0
     while count < 100:
             count = count + 1
             print "adding " + str(count) + " to queue"
             queue.put(count)

class RedisMQ(object):
    def __init__(self, app=None):
        if app is not None:
            self.init_app(app)
        else:
            self.app = None

        self._tasks = {}

    def init_app(self, app):
        name = app.config.setdefault('REDISMQ_NAME', 'mq')
        host = app.config.setdefault('REDISMQ_HOST', 'localhost')
        port = app.config.setdefault('REDISMQ_PORT', 6379)
        db = app.config.setdefault('REDISMQ_DB', 0)
        password = app.config.setdefault('REDISMQ_PASSWORD', None)

        app.config.get('REDISMQ_BLOCK', True),
        app.config.get('REDISMQ_BLOCK_TIMEOUT', 0)
        
        if not hasattr(app, 'extensions'):
            app.extensions = {}
        app.extensions['redismq'] = self

        self.app = app
        self._hq = HotQueue(name, host=host, port=port, db=db,
                            password=password)

    def task(self, func):
        
        func_name = "%s.%s" % (func.__module__, func.__name__)
        self._tasks[func_name] = func

        @wraps(func)
        def _func(*args, **kwargs):
            self._hq.put((func_name, args, kwargs))

        setattr(func, "async", _func)
        return func        

    def work(self, *args, **kwargs):
        kwargs.update({
            'block': self.app.config.get('REDISMQ_BLOCK', True),
            'timeout': self.app.config.get('REDISMQ_BLOCK_TIMEOUT', 0)
        })
        
        @self._hq.worker(*args, **kwargs)
        def _worker(msg):
            try:
                func_name, args, kwargs = msg
                self._tasks[func_name](*args, **kwargs)
            except Exception, e:
                pass

        return _worker()

def output_category(cat,data):
    global redis_enabled, appcfg

    pp.pprint(data)
    print "{cat} Entries: {size}".format(cat=cat,size=len(data))
    if appcfg['redis']:
        queue = HotQueue(
            appcfg['queue_name'], host=appcfg['redis_host'], port=appcfg['redis_port'], db=appcfg['redis_db']
        )
        for item in data:
            queue.put(item)

 def subscribe_greenlet(ps, connection_thread_id):
     queue = HotQueue("connection_thread_id_queue=" + str(connection_thread_id))
     for msg in queue.consume():
         try:
             cmd = json.loads(msg)
             if cmd['cmd'] == 'subscribe':
                 ps.subscribe('uuid=' + cmd['params'])
             elif cmd['cmd'] == 'subscribe':
                 ps.unsubscribe('uuid=' + cmd['params'])
         except:
             pass

def Main():

	config = file_read('config.json')
	config = json.loads(config)
	HOST = config['mongohost']
	PORT = config['port']
	DB_NAME = config['database_name']
	LISTENER_QUEUE = config['listener_queue']
	RESPONSE_QUEUE = config['response_queue']

	file_dir = config['repo_directory']
	client = MongoClient(HOST,PORT)
	
	db = client[DB_NAME]
	
	listen = HotQueue(LISTENER_QUEUE,serializer=json)

	response = HotQueue(RESPONSE_QUEUE,serializer=json)

	r = redis.StrictRedis(host='localhost', port=6379, db=0)

	print "\nPython-Redis-Listener-Started "

	for item in listen.consume():
		
		files = []
		for _file  in item['files']:
			files.append(diff_file(_file['name'],_file['path'],_file['isnew'],_file['tag'],db,item['repoid'],file_dir))
		
		commits = {
					'changes'  : files,
					'desc'     : item['desc'],
					'created'  : datetime.datetime.utcnow(),
					#'comment'  : [],
					'repo'     : {'id':ObjectId(item['repoid']),'slug':item['reposlug']},
					'user'     : {'id':ObjectId(item['userid']),'username':item['username']}
				}

		commitid = db.commits.insert(commits)
		
		db.repos.update({'_id':commits['repo']['id']},{'$push':{'commits':commitid}})
		
		db.users.update({'_id':commits['user']['id']},{'$push':{'commits':commitid}})
		
		responseobj= {'commitid':str(commitid),'userid':str(commits['user']['id'])}

		#response.put(responseobj)
		r.publish('cy-pullcommits', json.dumps(responseobj))
		
		print commits

class RedisInput(object):
    input_name = "redis"

    def __init__(self, host, queue, port=6379):
        self.host = host
        self.port = port
        self.queue = queue

    def handle_input(self):
        try:
            self.queue = HotQueue(self.queue, serializer=json, host=self.host, port=self.port, db=0)

            for data in self.queue.consume():
                for fmt in self.format_modules:
                    if fmt.bind and self.input_name in fmt.bind:
                        data = fmt.decode(data)

                self.output_threads.write(data)
        except Exception as err:
            raise EveConnectionError(err)

    def run(self, format_modules, output_modules):
        # Start output threads
        self.format_modules = format_modules
        self.output_modules = output_modules
        self.output_threads = OutputThreads(self.output_modules, self.format_modules)
        while True:
            try:
                self.handle_input()
            except EveConnectionError as err:
                logger.error("connection error in input handler %s: %r - retrying in 1 second" % (self.input_name, err))
                sleep(1)

def main():
    queue = HotQueue(main_config.INDEX_QUEUE, 
                     host=main_config.REDIS_HOST, 
                     port=main_config.REDIS_PORT)
    index = get_index(main_config.WHOOSH_INDEX_DIR)
    writer = BufferedWriter(index, limit=10)
    try:
        for doc_id in queue.consume():
            print "looking at {}".format(doc_id)
            doc = Document.query.get(doc_id)
            if doc:
                write_doc(doc, writer)
            else:
                print "no doc with doc_id {}".format(doc_id)
    finally:
       writer.close()

 def test_from_key(self):
     """Test the HotQueue.from_key static method"""
     redis = Redis()
     queues = {
         '_test:tens': range(10, 20),
         '_test:twenties': range(20, 30),
         '_test:thirties': range(30, 40)
     }
     for name, values in queues.items():
         q = HotQueue(name)
         q.put(*values)
     
     for key in redis.keys('hotqueue:_test:*'):
         q = HotQueue.from_key(key)
         self.assertEqual(
             list(q.consume(block=False)),
             queues[q.name]
         )

	def __init__(self, jukebox):
		cmd.Cmd.__init__(self)
		threading.Thread.__init__(self)
		self.jukebox = jukebox
		self.playlist = None
		self.track = None
		self.results = False
		
		self.queue_play = HotQueue("wekkie:play")

    def handle_input(self):
        try:
            self.queue = HotQueue(self.queue, serializer=json, host=self.host, port=self.port, db=0)

            for data in self.queue.consume():
                for fmt in self.format_modules:
                    if fmt.bind and self.input_name in fmt.bind:
                        data = fmt.decode(data)

                self.output_threads.write(data)
        except Exception as err:
            raise EveConnectionError(err)

def queue_processor():
        queue = HotQueue("postprocessing_queue")
        ocr = OCR.Engine()

        for uuid in queue.consume():
            print str(uuid)
            dm = DocumentManager()
            doc = dm.getDocument(uuid)
            print str(doc.url)
            url = urlparse(doc.url)
            filename = os.path.join(UPLOAD_FOLDER, url.path)
            m = magic.Magic()
            print filename + ' ' + str(m.id_filename(filename))

            if 'PDF' in  str(m.id_filename(filename)):
                pdf_text = PDFProcessor.extractContent(str(filename))
                cm = ContentDocument()
                cm.content = unicode(pdf_text, encoding='utf-8')
                cm.save()
                #ocr_text = ocr.interpret(str(filename))
                print pdf_text

    def configure(self, gconfig={}, **options):
        if self.running:
            raise SchedulerAlreadyRunningError

        config = combine_opts(gconfig, 'main.', options)
        self._config = config

        self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
        self.coalesce = asbool(config.pop('coalesce', True))
        self.daemonic = asbool(config.pop('daemonic', True))
        self.standalone = asbool(config.pop('standalone', False))

        timezone = config.pop('timezone', None)
        self.timezone = gettz(timezone) if isinstance(timezone, basestring) else timezone or tzlocal()

        # config threadpool
        threadpool_opts = combine_opts(config, 'threadpool.')
        self._worker_threadpool = ThreadPool(**threadpool_opts)

        # config jobstore
        jobstore_opts = combine_opts(config, 'jobstore.')
        self._job_store = SQLAlchemyJobStore(**jobstore_opts)

        # config syncqueue
        syncqueue_opts = combine_opts(config, 'syncqueue.')
        self._changes_queue = HotQueue(**syncqueue_opts)

        # config statstore
        statstore_opts = combine_opts(config, 'statstore.')
        self._stat_store = JobReporter(**statstore_opts)

        # config statqueue
        statqueue_opts = combine_opts(config, 'statqueue.')
        self._stats_queue = HotQueue(**statqueue_opts)

        # configure logger
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)