本文整理汇总了Python中pomp.core.engine.Pomp.queue方法的典型用法代码示例。如果您正苦于以下问题：Python Pomp.queue方法的具体用法？Python Pomp.queue怎么用？Python Pomp.queue使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pomp.core.engine.Pomp的用法示例。
在下文中一共展示了Pomp.queue方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_queue_get_requests_with_count
# 需要导入模块: from pomp.core.engine import Pomp [as 别名]
# 或者: from pomp.core.engine.Pomp import queue [as 别名]
def test_queue_get_requests_with_count(self):
class DummyDownloaderWithWorkers(DummyDownloader):
def get_workers_count(self):
return 5
class SimpleQueue(BaseQueue):
def __init__(self):
self.requests = []
def get_requests(self, count=None):
# Downloader can fetch only one request at moment
assert count == 5
try:
return self.requests.pop()
except IndexError:
return # empty queue
def put_requests(self, request):
self.requests.append(request)
pomp = Pomp(
downloader=DummyDownloaderWithWorkers(),
middlewares=(url_to_request_middl, ),
)
# override internal queue with own
pomp.queue = SimpleQueue()
pomp.pump(Crawler())示例2: test_queue_crawler
# 需要导入模块: from pomp.core.engine import Pomp [as 别名]
# 或者: from pomp.core.engine.Pomp import queue [as 别名]
def test_queue_crawler(self):
road = RoadPipeline()
class SimpleQueue(BaseQueue):
def __init__(self):
self.requests = []
def get_requests(self, count=None):
# because downloader without workers
assert count is None
try:
return self.requests.pop()
except IndexError:
return # empty queue
def put_requests(self, request):
self.requests.append(request)
pomp = Pomp(
downloader=DummyDownloader(),
middlewares=[url_to_request_middl],
pipelines=[
road,
],
)
# override internal queue with own
pomp.queue = SimpleQueue()
pomp.pump(Crawler())
assert set([item.url for item in road.collection]) == set([
'http://python.org/1',
'http://python.org/1/trash',
'http://python.org/2',
])