本文整理汇总了Python中pyspider.scheduler.task_queue.TaskQueue.put方法的典型用法代码示例。如果您正苦于以下问题：Python TaskQueue.put方法的具体用法？Python TaskQueue.put怎么用？Python TaskQueue.put使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pyspider.scheduler.task_queue.TaskQueue的用法示例。
在下文中一共展示了TaskQueue.put方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TestTaskQueue
# 需要导入模块: from pyspider.scheduler.task_queue import TaskQueue [as 别名]
# 或者: from pyspider.scheduler.task_queue.TaskQueue import put [as 别名]
class TestTaskQueue(unittest.TestCase):
@classmethod
def setUpClass(self):
self.task_queue = TaskQueue()
self.task_queue.rate = 100000
self.task_queue.burst = 100000
self.task_queue.processing_timeout = 0.2
self.task_queue.put('a3', 2, time.time() + 0.1)
self.task_queue.put('a1', 1)
self.task_queue.put('a2', 3)
def test_1_priority_queue(self):
self.assertEqual(self.task_queue.get(), 'a2')
def test_2_time_queue(self):
time.sleep(0.1)
self.task_queue.check_update()
self.assertEqual(self.task_queue.get(), 'a3')
self.assertEqual(self.task_queue.get(), 'a1')
def test_3_processing_queue(self):
time.sleep(0.1)
self.task_queue.check_update()
self.assertEqual(self.task_queue.get(), 'a2')
self.assertEqual(len(self.task_queue), 0)
def test_4_done(self):
self.task_queue.done('a2')
self.task_queue.done('a1')
time.sleep(0.1)
self.task_queue.check_update()
self.assertEqual(self.task_queue.get(), 'a3')
self.assertEqual(self.task_queue.get(), None)示例2: test_task_queue_in_time_order
# 需要导入模块: from pyspider.scheduler.task_queue import TaskQueue [as 别名]
# 或者: from pyspider.scheduler.task_queue.TaskQueue import put [as 别名]
def test_task_queue_in_time_order(self):
tq = TaskQueue(rate=300, burst=1000)
queues = dict()
tasks = dict()
for i in range(0, 100):
it = InQueueTask(str(i), priority=int(i // 10), exetime=0)
tq.put(it.taskid, it.priority, it.exetime)
if it.priority not in queues:
queues[it.priority] = Queue.Queue()
q = queues[it.priority] # type:Queue.Queue
q.put(it)
tasks[it.taskid] = it
six.print_('put, taskid=', it.taskid, 'priority=', it.priority, 'exetime=', it.exetime)
for i in range(0, 100):
task_id = tq.get()
task = tasks[task_id]
q = queues[task.priority] # type: Queue.Queue
expect_task = q.get()
self.assertEqual(task_id, expect_task.taskid)
self.assertEqual(task.priority, int(9 - i // 10))
six.print_('get, taskid=', task.taskid, 'priority=', task.priority, 'exetime=', task.exetime)
self.assertEqual(tq.size(), 100)
self.assertEqual(tq.priority_queue.qsize(), 0)
self.assertEqual(tq.processing.qsize(), 100)
for q in six.itervalues(queues): # type:Queue.Queue
self.assertEqual(q.qsize(), 0)
pass示例3: TestTaskQueue
# 需要导入模块: from pyspider.scheduler.task_queue import TaskQueue [as 别名]
# 或者: from pyspider.scheduler.task_queue.TaskQueue import put [as 别名]
class TestTaskQueue(unittest.TestCase):
@classmethod
def setUpClass(self):
self.task_queue = TaskQueue()
self.task_queue.rate = 100000
self.task_queue.burst = 100000
self.task_queue.processing_timeout = 0.5
def test_10_put(self):
self.task_queue.put('a3', 0, time.time() + 0.5)
self.task_queue.put('a4', 3, time.time() + 0.2)
self.task_queue.put('a2', 0)
self.task_queue.put('a1', 1)
self.assertEqual(self.task_queue.size(), 4)
def test_20_update(self):
self.task_queue.put('a2', 4)
self.assertEqual(self.task_queue.size(), 4)
self.task_queue.put('a3', 2, 0)
self.assertEqual(self.task_queue.size(), 4)
def test_30_get_from_priority_queue(self):
self.assertEqual(self.task_queue.get(), 'a2')
self.assertEqual(self.task_queue.size(), 4)
def test_40_time_queue_1(self):
self.task_queue.check_update()
self.assertEqual(self.task_queue.get(), 'a3')
self.assertEqual(self.task_queue.size(), 4)
def test_50_time_queue_2(self):
time.sleep(0.3)
self.task_queue.check_update()
self.assertEqual(self.task_queue.get(), 'a4')
self.assertEqual(self.task_queue.get(), 'a1')
self.assertEqual(self.task_queue.size(), 4)
def test_60_processing_queue(self):
time.sleep(0.5)
self.task_queue.check_update()
self.assertEqual(self.task_queue.get(), 'a2')
self.assertEqual(len(self.task_queue), 4)
self.assertEqual(self.task_queue.get(), 'a4')
self.assertEqual(self.task_queue.get(), 'a3')
self.assertEqual(self.task_queue.get(), 'a1')
self.assertEqual(len(self.task_queue), 4)
def test_70_done(self):
self.assertTrue(self.task_queue.done('a2'))
self.assertTrue(self.task_queue.done('a1'))
self.assertEqual(len(self.task_queue), 2)
self.assertTrue(self.task_queue.done('a4'))
self.assertTrue(self.task_queue.done('a3'))
self.assertEqual(len(self.task_queue), 0)示例4: test_time_queue
# 需要导入模块: from pyspider.scheduler.task_queue import TaskQueue [as 别名]
# 或者: from pyspider.scheduler.task_queue.TaskQueue import put [as 别名]
def test_time_queue(self):
six.print_('Test time queue order by time only')
tq = TaskQueue(rate=300, burst=1000)
fifo_queue = Queue.Queue()
interval = 5.0 / 1000
for i in range(0, 20):
it = InQueueTask(str(i), priority=int(i // 10), exetime=time.time() + (i + 1) * interval)
tq.put(it.taskid, it.priority, it.exetime)
fifo_queue.put(it)
six.print_('put, taskid=', it.taskid, 'priority=', it.priority, 'exetime=', it.exetime)
self.assertEqual(tq.priority_queue.qsize(), 0)
self.assertEqual(tq.processing.qsize(), 0)
self.assertEqual(tq.time_queue.qsize(), 20)
for i in range(0, 20):
t1 = fifo_queue.get()
t2 = tq.time_queue.get()
self.assertEqual(t1.taskid, t2.taskid)
six.print_('get, taskid=', t2.taskid, 'priority=', t2.priority, 'exetime=', t2.exetime)
self.assertEqual(tq.priority_queue.qsize(), 0)
self.assertEqual(tq.processing.qsize(), 0)
self.assertEqual(tq.time_queue.qsize(), 0)
queues = dict()
tasks = dict()
for i in range(0, 20):
priority = int(i // 10)
it = InQueueTask(str(i), priority=priority, exetime=time.time() + (i + 1) * interval)
tq.put(it.taskid, it.priority, it.exetime)
tasks[it.taskid] = it
if priority not in queues:
queues[priority] = Queue.Queue()
q = queues[priority]
q.put(it)
pass
self.assertEqual(tq.priority_queue.qsize(), 0)
self.assertEqual(tq.processing.qsize(), 0)
self.assertEqual(tq.time_queue.qsize(), 20)
time.sleep(20 * interval)
tq.check_update()
self.assertEqual(tq.priority_queue.qsize(), 20)
self.assertEqual(tq.processing.qsize(), 0)
self.assertEqual(tq.time_queue.qsize(), 0)
for i in range(0, 20):
taskid = tq.get()
t1 = tasks[taskid]
t2 = queues[t1.priority].get()
self.assertEqual(t1.taskid, t2.taskid)
self.assertEqual(tq.priority_queue.qsize(), 0)
self.assertEqual(tq.processing.qsize(), 20)
self.assertEqual(tq.time_queue.qsize(), 0)
pass