本文整理匯總了Python中frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler.enqueue_request方法的典型用法代碼示例。如果您正苦於以下問題：Python FronteraScheduler.enqueue_request方法的具體用法？Python FronteraScheduler.enqueue_request怎麽用？Python FronteraScheduler.enqueue_request使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler的用法示例。
在下文中一共展示了FronteraScheduler.enqueue_request方法的5個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: test_redirect_disabled_enqueue_requests
# 需要導入模塊: from frontera.contrib.scrapy.schedulers.frontier import FronteraScheduler [as 別名]
# 或者: from frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler import enqueue_request [as 別名]
def test_redirect_disabled_enqueue_requests(self):
settings = Settings()
settings['REDIRECT_ENABLED'] = False
crawler = FakeCrawler(settings)
fs = FronteraScheduler(crawler, manager=FakeFrontierManager)
fs.open(Spider)
assert fs.enqueue_request(rr1) is False
assert fs.enqueue_request(rr2) is False
assert fs.enqueue_request(rr3) is False示例2: test_enqueue_requests
# 需要導入模塊: from frontera.contrib.scrapy.schedulers.frontier import FronteraScheduler [as 別名]
# 或者: from frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler import enqueue_request [as 別名]
def test_enqueue_requests(self):
crawler = FakeCrawler()
fs = FronteraScheduler(crawler, manager=FakeFrontierManager)
fs.open(Spider)
assert fs.enqueue_request(r1) is True
assert fs.enqueue_request(r2) is True
assert fs.enqueue_request(r3) is True
assert set(seed.url for seed in fs.frontier.manager.seeds) == set([r1.url, r2.url, r3.url])
assert all([isinstance(seed, FRequest) for seed in fs.frontier.manager.seeds])
assert fs.stats_manager.stats.get_value('frontera/seeds_count') == 3示例3: test_redirect_enabled_enqueue_requests
# 需要導入模塊: from frontera.contrib.scrapy.schedulers.frontier import FronteraScheduler [as 別名]
# 或者: from frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler import enqueue_request [as 別名]
def test_redirect_enabled_enqueue_requests(self):
settings = Settings()
settings['REDIRECT_ENABLED'] = True
crawler = FakeCrawler(settings)
fs = FronteraScheduler(crawler, manager=FakeFrontierManager)
fs.open(Spider)
assert fs.enqueue_request(rr1) is True
assert fs.enqueue_request(rr2) is True
assert fs.enqueue_request(rr3) is True
assert set([request.url for request in fs._pending_requests]) == set([rr1.url, rr2.url, rr3.url])示例4: test_redirect_disabled_enqueue_requests
# 需要導入模塊: from frontera.contrib.scrapy.schedulers.frontier import FronteraScheduler [as 別名]
# 或者: from frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler import enqueue_request [as 別名]
def test_redirect_disabled_enqueue_requests(self):
settings = Settings()
settings['REDIRECT_ENABLED'] = False
crawler = FakeCrawler(settings)
fs = FronteraScheduler(crawler, manager=FakeFrontierManager)
fs.open(Spider)
assert fs.enqueue_request(rr1) is False
assert fs.enqueue_request(rr2) is False
assert fs.enqueue_request(rr3) is True
assert isinstance(fs.frontier.manager.seeds[0], FRequest)
assert len(fs.frontier.manager.seeds) == 1
assert fs.frontier.manager.seeds[0].url == rr3.url
assert fs.stats_manager.stats.get_value('frontera/seeds_count') == 1示例5: test_redirect_enabled_enqueue_requests
# 需要導入模塊: from frontera.contrib.scrapy.schedulers.frontier import FronteraScheduler [as 別名]
# 或者: from frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler import enqueue_request [as 別名]
def test_redirect_enabled_enqueue_requests(self):
settings = Settings()
settings['REDIRECT_ENABLED'] = True
crawler = FakeCrawler(settings)
fs = FronteraScheduler(crawler, manager=FakeFrontierManager)
fs.open(Spider)
assert fs.enqueue_request(rr1) is True
assert fs.enqueue_request(rr2) is True
assert fs.enqueue_request(rr3) is True
assert len(fs.frontier.manager.seeds) == 1
assert isinstance(fs.frontier.manager.seeds[0], FRequest)
assert fs.frontier.manager.seeds[0].url == rr3.url
assert set([request.url for request in fs._pending_requests]) == set([rr1.url, rr2.url])
assert all([isinstance(request, Request) for request in fs._pending_requests])
assert fs.stats_manager.stats.get_value('frontera/seeds_count') == 1
assert fs.stats_manager.stats.get_value('frontera/redirected_requests_count') == 2