本文整理匯總了Python中gevent.queue.JoinableQueue.get_nowait方法的典型用法代碼示例。如果您正苦於以下問題：Python JoinableQueue.get_nowait方法的具體用法？Python JoinableQueue.get_nowait怎麽用？Python JoinableQueue.get_nowait使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類gevent.queue.JoinableQueue的用法示例。
在下文中一共展示了JoinableQueue.get_nowait方法的1個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: BaseCrawler
# 需要導入模塊: from gevent.queue import JoinableQueue [as 別名]
# 或者: from gevent.queue.JoinableQueue import get_nowait [as 別名]
class BaseCrawler(object):
def __init__(self, requestHandler=BaseRequestHandler(),
parseHandler=BaseParseHandler(),
sheduler=BaseScheduler(),
pipeline=BasePipeline()):
self.requestHandler = requestHandler
self.parseHandler = parseHandler
self.sheduler = sheduler
self.pipeline = pipeline
self.task_queue = JoinableQueue()
self.response_queue = JoinableQueue()
self.tasks_cnt = 0
self.result_queue = JoinableQueue()
self.jobs_cnt = config.num_threads
self.start_time = time.time()
self.stop = False
def doScheduler(self):
"""Generate tasks, one thread
"""
logging.info('scheduler started!')
for task in self.sheduler.init_generator():
self.task_queue.put(task)
self.tasks_cnt += 1
while self.tasks_cnt > 0 and not self.stop:
gevent.sleep(config.new_task_check_time)
logging.info('scheduler finished! All task done.')
for i in xrange(config.num_threads):
self.task_queue.put(StopIteration)
def worker(self):
"""Fetch url and parse, config.num_threads threads
"""
task = self.task_queue.get()
cnt = config.error_retry_cnt
while task != StopIteration:
try:
#timeout = gevent.Timeout(config.TASK_TIMEOUT)
#timeout.start()
response = self.requestHandler.handle(task)
result, new_tasks = self.parseHandler.handle(response)
#timeout.cancel()
#if isinstance(result, collections.Iterable):
#if isinstance(result, list):
# for ret in result:
# self.result_queue.put(ret)
#else:
if result:
self.result_queue.put(result)
for task in new_tasks:
self.task_queue.put(task)
self.tasks_cnt += 1
#self.task_queue.task_done()
self.tasks_cnt -= 1
task = self.task_queue.get()
cnt = config.error_retry_cnt
except Exception as e:
try:
#timeout.cancel()
cnt -= 1
logging.exception(e)
if cnt <= 0:
#self.task_queue.task_done()
self.tasks_cnt -= 1
task = self.task_queue.get()
logging.error('task failed, try \033[31m%d\033[0m times! will not try' % (config.error_retry_cnt - cnt))
cnt = config.error_retry_cnt
#logging.exception('task failed!')
else:
logging.error('task failed, try \033[31m%d\033[0m times!' % (config.error_retry_cnt - cnt))
except Exception as e:
self.tasks_cnt -= 1
#self.jobs_cnt -= 1
raise
finally:
#timeout.cancel()
pass
self.jobs_cnt -= 1
def doPipeline(self):
while self.jobs_cnt > 0 or not self.result_queue.empty():
gevent.sleep(config.pipeline_sleeptime)
results = []
try:
while 1:
results.append(self.result_queue.get_nowait())
if len(results) > 100:
raise gevent.queue.Empty
except gevent.queue.Empty:
if results:
try:
self.pipeline.process(results)
except:
logging.exception('')
#logging.exception('')
except:
logging.exception('')
#.........這裏部分代碼省略.........