Python HotQueue.get方法代码示例

# 需要导入模块: from hotqueue import HotQueue [as 别名]
# 或者: from hotqueue.HotQueue import get [as 别名]
def main(argv) :
	global q

	featureName = argv[1]
	featureStatus = argv[2]
	featureServer = argv[3]

	q = HotQueue(('server_'+featureName), host="localhost", port=6379, db=0)
	r_server = redis.Redis('localhost')

	
	if featureStatus == "ON":
		q.put('http://127.0.0.1:'+featureServer)
		if(not r_server.exists(featureName)):
			r_server.set(featureName, '1')
		else:
			r_server.incr(featureName)
		print 'set key value is: ' + r_server.get(featureName) 
	else :
		k = list()	
		while(1) :
			x = q.get()
			print "Queue get : ", x
			if x is None : 
				break;
			else :
				if x == ('http://127.0.0.1:'+featureServer) :
					print("Removing : ",x)
					if(r_server.get(featureName) > 0):
						r_server.decr(featureName)
					print 'set key value is: ' + r_server.get(featureName) 
					break
				else:
					k.append(x)
		while len(k) != 0:
			x = k.pop(0)
			q.put(x)
	printQueue()
	print "Queue name : ",('server_'+featureName)

# 需要导入模块: from hotqueue import HotQueue [as 别名]
# 或者: from hotqueue.HotQueue import get [as 别名]
class HotQueueTestCase(unittest.TestCase):
    
    def setUp(self):
        """Create the queue instance before the test."""
        self.queue = HotQueue('testqueue')
    
    def tearDown(self):
        """Clear the queue after the test."""
        self.queue.clear()
    
    def test_arguments(self):
        """Test that HotQueue.__init__ accepts arguments correctly, and that
        the Redis key is correctly formed.
        """
        kwargs = {
            'name': "testqueue",
            'serializer': DummySerializer,
            'host': "localhost",
            'port': 6379,
            'db': 0}
        # Instantiate the HotQueue instance:
        self.queue = HotQueue(**kwargs)
        # Ensure that the properties of the instance are as expected:
        self.assertEqual(self.queue.name, kwargs['name'])
        self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
        self.assertEqual(self.queue.serializer, kwargs['serializer'])
        self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host'])
        self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host'])
        self.assertEqual(self.queue._HotQueue__redis.port, kwargs['port'])
        self.assertEqual(self.queue._HotQueue__redis.db, kwargs['db'])
        # Instantiate a HotQueue instance with only the required args:
        self.queue = HotQueue(kwargs['name'])
        # Ensure that the properties of the instance are as expected:
        self.assertEqual(self.queue.name, kwargs['name'])
        self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
        self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle or
                                                     # pickle, depending on the
                                                     # platform.
    
    def test_consume(self):
        """Test the consume generator method."""
        nums = [1, 2, 3, 4, 5, 6, 7, 8]
        # Test blocking with timeout:
        self.queue.put(*nums)
        msgs = []
        for msg in self.queue.consume(timeout=1):
            msgs.append(msg)
        self.assertEquals(msgs, nums)
        # Test non-blocking:
        self.queue.put(*nums)
        msgs = []
        for msg in self.queue.consume(block=False):
            msgs.append(msg)
        self.assertEquals(msgs, nums)
    
    def test_cleared(self):
        """Test for correct behaviour if the Redis list does not exist."""
        self.assertEquals(len(self.queue), 0)
        self.assertEquals(self.queue.get(), None)
    
    def test_get_order(self):
        """Test that messages are get in the same order they are put."""
        alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno']
        self.queue.put(alphabet[0], alphabet[1], alphabet[2])
        self.queue.put(alphabet[3])
        self.queue.put(alphabet[4])
        msgs = []
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        self.assertEquals(msgs, alphabet)
    
    def test_length(self):
        """Test that the length of a queue is returned correctly."""
        self.queue.put('a message')
        self.queue.put('another message')
        self.assertEquals(len(self.queue), 2)
    
    def test_worker(self):
        """Test the worker decorator."""
        colors = ['blue', 'green', 'red', 'pink', 'black']
        # Test blocking with timeout:
        self.queue.put(*colors)
        msgs = []
        @self.queue.worker(timeout=1)
        def appender(msg):
            msgs.append(msg)
        appender()
        self.assertEqual(msgs, colors)
        # Test non-blocking:
        self.queue.put(*colors)
        msgs = []
        @self.queue.worker(block=False)
        def appender(msg):
            msgs.append(msg)
        appender()
        self.assertEqual(msgs, colors)
    
#.........这里部分代码省略.........

# 需要导入模块: from hotqueue import HotQueue [as 别名]
# 或者: from hotqueue.HotQueue import get [as 别名]
class HotQueueTestCase(unittest.TestCase):
    
    def setUp(self):
        """Create the queue instance before the test."""
        self.queue = HotQueue('testqueue')
    
    def tearDown(self):
        """Clear the queue after the test."""
        self.queue.clear()
    
    def test_arguments(self):
        """Test that HotQueue.__init__ accepts arguments correctly, and that
        the Redis key is correctly formed.
        """
        kwargs = {
            'name': "testqueue",
            'serializer': DummySerializer,
            'host': "localhost",
            'port': 6379,
            'db': 0}
        # Instantiate the HotQueue instance:
        self.queue = HotQueue(**kwargs)
        # Ensure that the properties of the instance are as expected:
        self.assertEqual(self.queue.name, kwargs['name'])
        self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
        self.assertEqual(self.queue.serializer, kwargs['serializer'])
        # Instantiate a HotQueue instance with only the required args:
        self.queue = HotQueue(kwargs['name'])
        # Ensure that the properties of the instance are as expected:
        self.assertEqual(self.queue.name, kwargs['name'])
        self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name'])
        self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle
                                                         # or pickle, depending
                                                         # on the platform.
    
    def test_consume(self):
        """Test the consume generator method."""
        nums = [1, 2, 3, 4, 5, 6, 7, 8]
        # Test blocking with timeout:
        nums_added = self.queue.put(*nums)
        msgs = []
        for msg in self.queue.consume(timeout=1):
            msgs.append(msg)
        self.assertEqual(msgs, nums_added)
        for msg in msgs:
            self.queue.ack(msg.get_reservationId())
        # Test non-blocking:
        nums_added = self.queue.put(*nums)
        msgs = []
        for msg in self.queue.consume(block=False):
            msgs.append(msg)
        self.assertEqual(msgs, nums_added)
        for msg in msgs:
            self.queue.ack(msg.get_reservationId())

    def test_nack(self):
        """Test the consume generator method."""
        nums_added = self.queue.put("1")
        msg = self.queue.get() #1
        self.assertEqual(msg, nums_added[0])
        self.queue.nack(msg.get_reservationId())
        msg = self.queue.get() #2
        self.assertEqual(msg, nums_added[0])
        self.queue.nack(msg.get_reservationId())
        msg = self.queue.get() #3
        if msg:
            self.assertEqual(msg, nums_added[0])
            self.queue.ack(msg.get_reservationId())
        self.assertEqual(msg, nums_added[0])
        self.assertEqual(len(self.queue), 0)
        self.assertEqual(msg.get_deliveryCount(),3) #3
    
    def test_cleared(self):
        """Test for correct behaviour if the Redis list does not exist."""
        self.assertEqual(len(self.queue), 0)
        self.assertEqual(self.queue.get(), None)
    
    def test_get_order(self):
        """Test that messages are get in the same order they are put."""
        alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno']
        msg_added = []
        msg_added.extend(self.queue.put(alphabet[0], alphabet[1], alphabet[2]))
        msg_added.extend(self.queue.put(alphabet[3]))
        msg_added.extend(self.queue.put(alphabet[4]))
        msgs = []
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        self.assertEqual(msgs, msg_added)
        for msg in msgs:
            self.queue.ack(msg.get_reservationId())

    def test_length(self):
        """Test that the length of a queue is returned correctly."""
        self.queue.put('a message')
        self.queue.put('another message')
        self.assertEqual(len(self.queue), 2)
    
#.........这里部分代码省略.........

# 需要导入模块: from hotqueue import HotQueue [as 别名]
# 或者: from hotqueue.HotQueue import get [as 别名]
class LocalScheduler(object):

    _stopped = False
    _main_thread = None

    #init worker thread pool，reporter thread，updater thread
    def __init__(self, gconfig={}, **options):
        self._wakeup = Event()
        self._job_store = None
        self._stat_store = None
        self._jobs     = {}
        self.logger   = None
        self._stats_queue = None
        self._changes_queue = None

        self._jobs_locks   = {}
        self._jobs_lock = Lock()
        self._log_queue_lock = Lock()


        self._worker_threadpool = None
        self._reporter_thread   = None
        self._main_thread       = None
        self._updater_thread    = None
        self._monitor_thread    = None

        self.configure(gconfig, **options)

    def configure(self, gconfig={}, **options):
        if self.running:
            raise SchedulerAlreadyRunningError

        config = combine_opts(gconfig, 'main.', options)
        self._config = config

        self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
        self.coalesce = asbool(config.pop('coalesce', True))
        self.daemonic = asbool(config.pop('daemonic', True))
        self.standalone = asbool(config.pop('standalone', False))

        timezone = config.pop('timezone', None)
        self.timezone = gettz(timezone) if isinstance(timezone, basestring) else timezone or tzlocal()

        # config threadpool
        threadpool_opts = combine_opts(config, 'threadpool.')
        self._worker_threadpool = ThreadPool(**threadpool_opts)

        # config jobstore
        jobstore_opts = combine_opts(config, 'jobstore.')
        self._job_store = SQLAlchemyJobStore(**jobstore_opts)

        # config syncqueue
        syncqueue_opts = combine_opts(config, 'syncqueue.')
        self._changes_queue = HotQueue(**syncqueue_opts)

        # config statstore
        statstore_opts = combine_opts(config, 'statstore.')
        self._stat_store = JobReporter(**statstore_opts)

        # config statqueue
        statqueue_opts = combine_opts(config, 'statqueue.')
        self._stats_queue = HotQueue(**statqueue_opts)

        # configure logger
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)



    def start(self):
        if self.running:
            raise SchedulerAlreadyRunningError

        self.load_jobs()

        self._stopped = False

        if self.standalone:
            self._main_loop()
        else:
            self._main_thread = Thread(target = self._main_loop, name = 'main')
            self._main_thread.setDaemon(self.daemonic)
            self._main_thread.start()
            print 'main thread is startted'

            self._updater_thread = Thread(target = self._sync_changes, name = 'update')
            self._updater_thread.setDaemon(self.daemonic)
            self._updater_thread.start()
            print 'update thread is started'

            self._stater_thread = Thread(target = self._stat_runs, name = 'stat')
            self._stater_thread.setDaemon(self.daemonic)
            self._stater_thread.start()
            print 'stat thread is started'

    def shutdown(self, shutdown_threadpool=True, close_jobstore=True):
        if not self.running:
            return 
        self._stopped = True
        self._wakeup.set()
#.........这里部分代码省略.........

# 需要导入模块: from hotqueue import HotQueue [as 别名]
# 或者: from hotqueue.HotQueue import get [as 别名]
class HotQueueTestCase(unittest.TestCase):
    def setUp(self):
        """Create the queue instance before the test."""
        self.queue = HotQueue("testqueue")

    def tearDown(self):
        """Clear the queue after the test."""
        self.queue.clear()

    def test_arguments(self):
        """Test that HotQueue.__init__ accepts arguments correctly, and that
        the Redis key is correctly formed.
        """
        kwargs = {"name": "testqueue", "serializer": DummySerializer, "host": "localhost", "port": 6379, "db": 0}
        # Instantiate the HotQueue instance:
        self.queue = HotQueue(**kwargs)
        # Ensure that the properties of the instance are as expected:
        self.assertEqual(self.queue.name, kwargs["name"])
        self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs["name"])
        self.assertEqual(self.queue.serializer, kwargs["serializer"])
        self.assertEqual(self.queue._HotQueue__redis.host, kwargs["host"])
        self.assertEqual(self.queue._HotQueue__redis.host, kwargs["host"])
        self.assertEqual(self.queue._HotQueue__redis.port, kwargs["port"])
        self.assertEqual(self.queue._HotQueue__redis.db, kwargs["db"])
        # Instantiate a HotQueue instance with only the required args:
        self.queue = HotQueue(kwargs["name"])
        # Ensure that the properties of the instance are as expected:
        self.assertEqual(self.queue.name, kwargs["name"])
        self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs["name"])
        self.assertTrue(self.queue.serializer is pickle)  # Defaults to cPickle
        # or pickle, depending
        # on the platform.

    def test_consume(self):
        """Test the consume generator method."""
        nums = [1, 2, 3, 4, 5, 6, 7, 8]
        # Test blocking with timeout:
        self.queue.put(*nums)
        msgs = []
        for msg in self.queue.consume(timeout=1):
            msgs.append(msg)
        self.assertEqual(msgs, nums)
        # Test non-blocking:
        self.queue.put(*nums)
        msgs = []
        for msg in self.queue.consume(block=False):
            msgs.append(msg)
        self.assertEqual(msgs, nums)

    def test_cleared(self):
        """Test for correct behaviour if the Redis list does not exist."""
        self.assertEqual(len(self.queue), 0)
        self.assertEqual(self.queue.get(), None)

    def test_get_order(self):
        """Test that messages are get in the same order they are put."""
        alphabet = ["abc", "def", "ghi", "jkl", "mno"]
        self.queue.put(alphabet[0], alphabet[1], alphabet[2])
        self.queue.put(alphabet[3])
        self.queue.put(alphabet[4])
        msgs = []
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        self.assertEqual(msgs, alphabet)

    def test_length(self):
        """Test that the length of a queue is returned correctly."""
        self.queue.put("a message")
        self.queue.put("another message")
        self.assertEqual(len(self.queue), 2)

    def test_worker(self):
        """Test the worker decorator."""
        colors = ["blue", "green", "red", "pink", "black"]
        # Test blocking with timeout:
        self.queue.put(*colors)
        msgs = []

        @self.queue.worker(timeout=1)
        def appender(msg):
            msgs.append(msg)

        appender()
        self.assertEqual(msgs, colors)
        # Test non-blocking:
        self.queue.put(*colors)
        msgs = []

        @self.queue.worker(block=False)
        def appender(msg):
            msgs.append(msg)

        appender()
        self.assertEqual(msgs, colors)
        # Test decorating a class method:
        self.queue.put(*colors)
        msgs = []
#.........这里部分代码省略.........

# 需要导入模块: from hotqueue import HotQueue [as 别名]
# 或者: from hotqueue.HotQueue import get [as 别名]
			logger.exception('bli social shares total: ' + str(self.total))
			self.loop.stop()



try:
	conn = S3Connection(awsAccessKey, awsSecretKey)
	buckets = conn.get_bucket('aws-publicdatasets')

except Exception, ex:
	logger.exception("Fatal Error - Failed to connect to S3")
	sys.exit()


arcQueue = HotQueue('arc', host=arcQueueServer, port=arcQueuePort, db=0)
arcKey = arcQueue.get()

# arcKey is received from HotQueue
#arcKey = '1262847559577_0.arc.gz'


logger.exception('crawl beginning')

#sentinel loop to continue processing queue till it's empty
while arcKey != None:
	logger.exception('fetching arc: ' + arcKey)
	

	
	# The S3 API Version streams the file in, decompresses the GZIP, and then converts it to a file object for iteration

# 需要导入模块: from hotqueue import HotQueue [as 别名]
# 或者: from hotqueue.HotQueue import get [as 别名]
class HotQueueTestCase(unittest.TestCase):
    
    def setUp(self):
        """Create the queue instance before the test."""
        self.queue = HotQueue('testqueue')
    
    def tearDown(self):
        """Clear the queue after the test."""
        self.queue.clear()
    
    def test_consume(self):
        """Test the consume generator method."""
        nums = [1, 2, 3, 4, 5, 6, 7, 8]
        # Test blocking with timeout:
        self.queue.put(*nums)
        msgs = []
        for msg in self.queue.consume(timeout=1):
            msgs.append(msg)
        self.assertEquals(msgs, nums)
        # Test non-blocking:
        self.queue.put(*nums)
        msgs = []
        for msg in self.queue.consume(block=False):
            msgs.append(msg)
        self.assertEquals(msgs, nums)
    
    def test_cleared(self):
        """Test for correct behaviour if the Redis list does not exist."""
        self.assertEquals(len(self.queue), 0)
        self.assertEquals(self.queue.get(), None)
    
    def test_get_order(self):
        """Test that messages are get in the same order they are put."""
        alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno']
        self.queue.put(alphabet[0], alphabet[1], alphabet[2])
        self.queue.put(alphabet[3])
        self.queue.put(alphabet[4])
        msgs = []
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        msgs.append(self.queue.get())
        self.assertEquals(msgs, alphabet)
    
    def test_length(self):
        """Test that the length of a queue is returned correctly."""
        self.queue.put('a message')
        self.queue.put('another message')
        self.assertEquals(len(self.queue), 2)
    
    def test_worker(self):
        """Test the worker decorator."""
        colors = ['blue', 'green', 'red', 'pink', 'black']
        # Test blocking with timeout:
        self.queue.put(*colors)
        msgs = []
        @self.queue.worker(timeout=1)
        def appender(msg):
            msgs.append(msg)
        appender()
        self.assertEqual(msgs, colors)
        # Test non-blocking:
        self.queue.put(*colors)
        msgs = []
        @self.queue.worker(block=False)
        def appender(msg):
            msgs.append(msg)
        appender()
        self.assertEqual(msgs, colors)
    
    def test_threaded(self):
        """Threaded test of put and consume methods."""
        msgs = []
        def put():
            for num in range(3):
                self.queue.put('message %d' % num)
                sleep(0.1)
        def consume():
            for msg in self.queue.consume(timeout=1):
                msgs.append(msg)
        putter = threading.Thread(target=put)
        consumer = threading.Thread(target=consume)
        putter.start()
        consumer.start()
        for thread in [putter, consumer]:
            thread.join()
        self.assertEqual(msgs, ['message 0', 'message 1', 'message 2'])

# 需要导入模块: from hotqueue import HotQueue [as 别名]
# 或者: from hotqueue.HotQueue import get [as 别名]
 def get(self):
     queue = HotQueue("myqueue", host="localhost", port=6379, db=0)
     self.write('Hello from tornado' + str(queue.get()))