Python models.Queue类代码示例

    def test_job_error_method_should_be_called(self):
        class ExceptionWithCode(Exception):
            def __init__(self, message, code):
                super(ExceptionWithCode, self).__init__(message)
                self.message = message
                self.code = code

        def callback(job, queue):
            raise ExceptionWithCode('foobar', 42)

        job1 = Job.add_job(identifier='job:1', queue_name='test')
        queue = Queue.get_queue(name='test')
        worker = Worker(name='test', max_loops=1)  # no callback
        worker.run()

        self.assertEqual(job1.status.hget(), STATUSES.ERROR)
        self.assertIn(job1.get_pk(), queue.errors.lmembers())
        self.assertEqual(len(Error.collection()), 1)
        error = Error.get(identifier='job:1')
        self.assertEqual(error.message.hget(), 'You must implement your own action')
        self.assertEqual(error.code.hget(), None)

        job2 = Job.add_job(identifier='job:2', queue_name='test')
        queue = Queue.get_queue(name='test')
        worker = Worker(name='test', max_loops=1, callback=callback)  # callback with exception
        worker.run()

        self.assertEqual(job2.status.hget(), STATUSES.ERROR)
        self.assertIn(job2.get_pk(), queue.errors.lmembers())
        self.assertEqual(len(Error.collection()), 2)
        error = Error.get(identifier='job:2')
        self.assertEqual(error.message.hget(), 'foobar')
        self.assertEqual(error.code.hget(), '42')

 def test_existing_queue_should_be_returned(self):
     # create one
     Queue.get_queue(name='test')
     # and get it
     count_before = self.count_queues()
     Queue.get_queue(name='test')
     count_after = self.count_queues()
     self.assertEqual(count_after, count_before)

 def test_count_delayed_jobs_should_return_the_number_of_delayed_jobs(self):
     self.assertEqual(Queue.count_delayed_jobs('test'), 0)
     Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
     self.assertEqual(Queue.count_delayed_jobs('test'), 1)
     Job.add_job(identifier='job:2', queue_name='test')
     self.assertEqual(Queue.count_delayed_jobs('test'), 1)
     Job.add_job(identifier='foo:1', queue_name='foo', delayed_for=5)
     Job.add_job(identifier='bar:1', queue_name='bar', delayed_for=5)
     self.assertEqual(Queue.count_delayed_jobs(['foo', 'bar']), 2)
     self.assertEqual(Queue.count_delayed_jobs(['test', 'foo', 'bar']), 3)

 def test_a_job_in_error_could_be_requeued(self):
     job = Job(identifier='job:1', status=STATUSES.ERROR)
     job.requeue('test')
     queue = Queue.get_queue('test')
     self.assertEqual(queue.waiting.llen(), 1)
     self.assertEqual(queue.delayed.zcard(), 0)
     self.assertEqual(job.status.hget(), STATUSES.WAITING)

    def test_adding_a_job_with_delayed_until_not_in_the_future_should_add_the_job_in_the_waiting_list(self):
        queue = Queue.get_queue(name='test')

        job = Job.add_job(identifier='job:1', queue_name='test', delayed_until=datetime.utcnow() - timedelta(seconds=5))
        self.assertEqual(job.status.hget(), STATUSES.WAITING)
        self.assertEqual(queue.waiting.llen(), 1)
        self.assertEqual(queue.delayed.zcard(), 0)

    def test_wait_for_job_should_respond_with_queue_and_job_when_a_queue_is_not_empty(self):
        queue = Queue.get_queue('test')
        worker = Worker('test')
        worker.test_content = None

        class Thread(threading.Thread):
            def run(self):
                worker.update_keys()
                queue, job = worker.wait_for_job()
                worker.test_content = queue, job

        # start listening
        thread = Thread()
        thread.start()
        time.sleep(0.1)

        # no job, nothing received
        self.assertIsNone(worker.test_content)

        # add a job
        job = Job.add_job(queue_name='test', identifier='job:1')
        time.sleep(0.1)

        # info should be received
        self.assertEqual(worker.test_content[0].get_pk(), queue.get_pk())
        self.assertTrue(isinstance(worker.test_content[0], Queue))
        self.assertEqual(worker.test_content[1].get_pk(), job.get_pk())
        self.assertTrue(isinstance(worker.test_content[1], Job))

        # job must no be in the queue anymore
        self.assertNotIn(job.get_pk(), queue.waiting.lmembers())

    def test_adding_an_existing_job_with_lower_priority_should_do_nothing(self):
        job1 = Job.add_job(identifier='job:1', queue_name='test', priority=3)
        job2 = Job.add_job(identifier='job:1', queue_name='test', priority=1)

        # only one job
        self.assertEqual(job1.ident, job2.ident)
        # is in high priority queue
        queue = Queue.get_queue(name='test', priority=3)
        self.assertEqual(queue.waiting.llen(), 1)
        # nothing in high priority queue
        queue = Queue.get_queue(name='test', priority=1)
        self.assertEqual(queue.waiting.llen(), 0)

        # we should still have original priority and status
        self.assert_job_status_and_priority(job1, STATUSES.WAITING, '3')
        # idem for job2 (which is, in fact, job1)
        self.assert_job_status_and_priority(job2, STATUSES.WAITING, '3')

    def test_adding_an_existing_job_with_higher_priority_should_change_its_queue(self):
        job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
        queue1 = Queue.get_queue(name='test', priority=1)
        job2 = Job.add_job(identifier='job:1', queue_name='test', priority=2)
        queue2 = Queue.get_queue(name='test', priority=2)

        # only one job
        self.assertEqual(job1.ident, job2.ident)
        # not anymore in queue with priority 1
        self.assertEqual(queue1.waiting.llen(), 0)
        # but now in queue with priority 2
        self.assertEqual(queue2.waiting.llen(), 1)

        # the new prioriy must be stored, and status should still be waiting
        self.assert_job_status_and_priority(job1, STATUSES.WAITING, '2')
        # idem for job2 (which is, in fact, job1)
        self.assert_job_status_and_priority(job2, STATUSES.WAITING, '2')

    def test_add_job_without_queue_name_should_use_the_class_one(self):
        class JobWithQueueName(Job):
            queue_name = 'test'

        JobWithQueueName.add_job(identifier='job:1')
        queue = Queue.get_queue('test')

        self.assertEqual(queue.waiting.llen(), 1)

    def test_requeue_delayed_jobs_put_back_ready_delayed_jobs_to_the_waiting_list(self):
        queue = Queue.get_queue(name='test')

        job1 = Job.add_job(identifier='job:1', queue_name='test', delayed_for=10)
        job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)

        self.assertEqual(Queue.count_waiting_jobs('test'), 0)
        self.assertEqual(Queue.count_delayed_jobs('test'), 2)

        self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
        self.assertEqual(job2.status.hget(), STATUSES.DELAYED)

        # must not move any jobs, too soon
        queue.requeue_delayed_jobs()
        self.assertEqual(Queue.count_waiting_jobs('test'), 0)
        self.assertEqual(Queue.count_delayed_jobs('test'), 2)

        self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
        self.assertEqual(job2.status.hget(), STATUSES.DELAYED)

        sleep(1)

        # now we should have one job in the waiting list
        queue.requeue_delayed_jobs()
        self.assertEqual(Queue.count_waiting_jobs('test'), 1)
        self.assertEqual(Queue.count_delayed_jobs('test'), 1)

        self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
        self.assertEqual(job2.status.hget(), STATUSES.WAITING)

        waiting_jobs = queue.waiting.lmembers()
        self.assertEqual(waiting_jobs, [job2.ident])
        delayed_jobs = queue.delayed.zrange(0, -1, withscores=True)
        self.assertEqual(delayed_jobs[0][0], job1.ident)

    def test_using_a_subclass_of_queue_should_work(self):
        class TestQueue(Queue):
            namespace = 'test_using_a_subclass_of_queue_should_work'

        class TestJob(Job):
            namespace = 'test_using_a_subclass_of_queue_should_work'

        # not specifying queue_model should use the default Queue model
        default_queue = Queue.get_queue('test1')
        queue = TestQueue.get_queue('test1')
        Job.add_job(identifier='job:1', queue_name='test1')
        self.assertEqual(queue.waiting.llen(), 0)
        self.assertEqual(default_queue.waiting.llen(), 1)

        # idem with a subclass of job
        default_queue = Queue.get_queue('test2')
        queue = TestQueue.get_queue('test2')
        TestJob.add_job(identifier='job:2', queue_name='test2')
        self.assertEqual(queue.waiting.llen(), 0)
        self.assertEqual(default_queue.waiting.llen(), 1)

        # specifiying a queue_model in add_job should use the wanted queue
        default_queue = Queue.get_queue('test3')
        queue = TestQueue.get_queue('test3')
        Job.add_job(identifier='job:3', queue_name='test3', queue_model=TestQueue)
        self.assertEqual(queue.waiting.llen(), 1)
        self.assertEqual(default_queue.waiting.llen(), 0)

        # idem with a subclass of job
        default_queue = Queue.get_queue('test4')
        queue = TestQueue.get_queue('test4')
        TestJob.add_job(identifier='job:4', queue_name='test4', queue_model=TestQueue)
        self.assertEqual(queue.waiting.llen(), 1)
        self.assertEqual(default_queue.waiting.llen(), 0)

        # now test with a queue_model defined in the job class
        class TestJobWithQueueModel(Job):
            namespace = 'test_using_a_subclass_of_queue_should_work'
            queue_model = TestQueue

        default_queue = Queue.get_queue('test5')
        queue = TestQueue.get_queue('test5')
        TestJobWithQueueModel.add_job(identifier='job:5', queue_name='test5')
        self.assertEqual(queue.waiting.llen(), 1)
        self.assertEqual(default_queue.waiting.llen(), 0)

    def test_enqueue_or_delay_without_queue_name_should_use_the_class_one(self):
        class JobWithQueueName3(Job):
            queue_name = 'test'

        job = JobWithQueueName3(identifier='job:1')
        job.enqueue_or_delay(delayed_until=datetime.utcnow()+timedelta(seconds=5))

        queue = Queue.get_queue('test')
        self.assertEqual(queue.delayed.zcard(), 1)

    def test_adding_a_job_should_create_a_queue_with_the_job(self):
        job = Job.add_job(identifier='job:1', queue_name='test', priority=5)

        # count queues
        keys = Queue.get_waiting_keys('test')
        self.assertEqual(len(keys), 1)

        # get the new queue, should not create it (number of keys should be 1)
        queue = Queue.get_queue(name='test', priority=5)
        keys = Queue.get_waiting_keys('test')
        self.assertEqual(len(keys), 1)

        # check that the job is in the queue
        jobs = queue.waiting.lrange(0, -1)
        self.assertEqual(jobs, [job.ident])

        # ... with the correct status and priority
        self.assert_job_status_and_priority(job, STATUSES.WAITING, '5')

    def test_prepending_an_existing_job_should_move_it_at_the_beginning(self):
        queue = Queue.get_queue(name='test', priority=1)

        job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
        job2 = Job.add_job(identifier='job:2', queue_name='test', priority=1)
        self.assertEqual(queue.waiting.lmembers(), [job1.ident, job2.ident])

        Job.add_job(identifier='job:2', queue_name='test', priority=1, prepend=True)
        self.assertEqual(queue.waiting.lmembers(), [job2.ident, job1.ident])

 def test_requeuing_a_job_with_a_datetime_should_put_it_in_the_delayed_lits(self):
     job = Job(identifier='job:1', status=STATUSES.ERROR)
     queue = Queue.get_queue('test')
     self.assertEqual(queue.waiting.llen(), 0)
     self.assertEqual(queue.delayed.zcard(), 0)
     job.requeue('test', delayed_until=datetime.utcnow()+timedelta(seconds=5))
     self.assertEqual(queue.waiting.llen(), 0)
     self.assertEqual(queue.delayed.zcard(), 1)
     self.assertEqual(job.status.hget(), STATUSES.DELAYED)