本文整理匯總了Python中frontera.settings.Settings.STRATEGY方法的典型用法代碼示例。如果您正苦於以下問題：Python Settings.STRATEGY方法的具體用法？Python Settings.STRATEGY怎麽用？Python Settings.STRATEGY使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類frontera.settings.Settings的用法示例。
在下文中一共展示了Settings.STRATEGY方法的4個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: test_blocking_middleware
# 需要導入模塊: from frontera.settings import Settings [as 別名]
# 或者: from frontera.settings.Settings import STRATEGY [as 別名]
def test_blocking_middleware(self):
settings = Settings()
settings.BACKEND = 'tests.mocks.components.FakeBackend'
settings.MIDDLEWARES = ['frontera.contrib.middlewares.domain.DomainMiddleware',
'frontera.contrib.middlewares.fingerprint.UrlFingerprintMiddleware',
'tests.mocks.components.FakeMiddleware',
'tests.mocks.components.FakeMiddlewareModifySeeds',
'tests.mocks.components.FakeMiddlewareBlocking',
'tests.mocks.components.FakeMiddlewareModifyResponse',
'tests.mocks.components.FakeMiddlewareModifyLinks']
settings.CANONICAL_SOLVER = 'tests.mocks.components.FakeCanonicalSolver'
settings.STRATEGY = 'tests.mocks.components.CrawlingStrategy'
fm = LocalFrontierManager.from_settings(settings)
SEEDS_FILE.seek(0)
fm.add_seeds(SEEDS_FILE)
response = Response(r1.url, request=r1)
fm.page_crawled(response)
fm.links_extracted(r1, links=[r2])
fm.request_error(r3, 'error')
#the seeds, responses, links and errors have not reached the backend.
assert [len(list) for list in fm.backend.lists] == [0]*4
#the 3 seeds reach the first three middlewares.
assert [len(fm.middlewares[i].requests) for i in range(2, 5)] == [3]*3
#the error, response and link reached the first three middlewares.
assert [[len(list) for list in fm.middlewares[i].lists[1:]] for i in range(2, 5)] == [[1]*3]*3
#the values do not reach the bottom 2 middlewares and the canonical solver.
assert [[len(list) for list in fm.middlewares[i].lists] for i in range(5, 7)] == [[0]*4]*2
assert [len(list) for list in fm.canonicalsolver.lists] == [0]*4示例2: sw_setup_add_seeds
# 需要導入模塊: from frontera.settings import Settings [as 別名]
# 或者: from frontera.settings.Settings import STRATEGY [as 別名]
def sw_setup_add_seeds(self):
settings = Settings()
settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed'
settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus'
settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100
settings.STRATEGY = 'tests.mocks.components.CrawlingStrategy'
return StrategyWorker(settings, True)示例3: sw_setup_filtered_links
# 需要導入模塊: from frontera.settings import Settings [as 別名]
# 或者: from frontera.settings.Settings import STRATEGY [as 別名]
def sw_setup_filtered_links(self):
settings = Settings()
settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed'
settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus'
settings.STRATEGY = 'tests.test_worker_strategy.FilteredLinksCrawlingStrategy'
settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100
return StrategyWorker(settings, False)示例4: strategy
# 需要導入模塊: from frontera.settings import Settings [as 別名]
# 或者: from frontera.settings.Settings import STRATEGY [as 別名]
def strategy(self):
settings = Settings()
settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed'
settings.STRATEGY = 'tests.test_strategy.DummyCrawlingStrategy'
manager = WorkerFrontierManager.from_settings(settings, db_worker=False, strategy_worker=True)
stream = MessageBusStream()
states = MemoryStates(10)
states_ctx = StatesContext(states)
return manager.strategy