本文整理汇总了Python中holmes.models.Page.get_next_job方法的典型用法代码示例。如果您正苦于以下问题：Python Page.get_next_job方法的具体用法？Python Page.get_next_job怎么用？Python Page.get_next_job使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类holmes.models.Page的用法示例。
在下文中一共展示了Page.get_next_job方法的8个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_can_get_next_job_when_domain_limited
# 需要导入模块: from holmes.models import Page [as 别名]
# 或者: from holmes.models.Page import get_next_job [as 别名]
def test_can_get_next_job_when_domain_limited(self):
self.db.query(Domain).delete()
self.db.query(Page).delete()
domain_a = DomainFactory.create()
domain_b = DomainFactory.create()
LimiterFactory.create(url=domain_a.url, value=2)
pages_a = []
pages_b = []
workers = []
for i in range(10):
for j in range(2):
workers.append(WorkerFactory.create())
pages_a.append(PageFactory.create(domain=domain_a, url="%s/%d.html" % (domain_a.url, i), score=i * 10))
pages_b.append(PageFactory.create(domain=domain_b, url="%s/%d.html" % (domain_b.url, i), score=i))
# first one should not be limited
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1,
avg_links_per_page=10
)
expect(next_job).not_to_be_null()
expect(next_job['page']).to_equal(str(pages_a[-1].uuid))
workers[0].current_url = next_job['url']
self.db.flush()
# second one should be limited (2 / 10 = 0.2, rounded up = 1 job at a time)
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1
)
expect(next_job).not_to_be_null()
expect(next_job['page']).to_equal(str(pages_b[-1].uuid))示例2: test_increases_page_score_when_lambda_is_top_page
# 需要导入模块: from holmes.models import Page [as 别名]
# 或者: from holmes.models.Page import get_next_job [as 别名]
def test_increases_page_score_when_lambda_is_top_page(self):
WorkerFactory.create()
page = PageFactory.create()
page2 = PageFactory.create()
settings = Settings.instance(self.db)
settings.lambda_score = 10000
Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1
)
self.db.refresh(page)
self.db.refresh(page2)
expect(page.score).to_equal(5000)
expect(page2.score).to_equal(5000)示例3: test_can_get_next_job
# 需要导入模块: from holmes.models import Page [as 别名]
# 或者: from holmes.models.Page import get_next_job [as 别名]
def test_can_get_next_job(self):
page = PageFactory.create()
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1
)
expect(next_job).not_to_be_null()
expect(next_job['page']).to_equal(str(page.uuid))示例4: test_can_get_next_job_when_expired
# 需要导入模块: from holmes.models import Page [as 别名]
# 或者: from holmes.models.Page import get_next_job [as 别名]
def test_can_get_next_job_when_expired(self):
page = PageFactory.create(last_review_date=datetime(2010, 10, 10, 10, 10, 10))
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1
)
expect(next_job).not_to_be_null()
expect(next_job['page']).to_equal(str(page.uuid))示例5: test_get_next_job_does_not_get_from_inactive_domains
# 需要导入模块: from holmes.models import Page [as 别名]
# 或者: from holmes.models.Page import get_next_job [as 别名]
def test_get_next_job_does_not_get_from_inactive_domains(self):
domain = DomainFactory.create(is_active=False)
PageFactory.create(domain=domain)
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1
)
expect(next_job).to_be_null()示例6: test_can_get_next_job
# 需要导入模块: from holmes.models import Page [as 别名]
# 或者: from holmes.models.Page import get_next_job [as 别名]
def test_can_get_next_job(self):
domain = DomainFactory.create()
pages = []
for i in range(20):
WorkerFactory.create()
pages.append(PageFactory.create(
domain=domain,
score=float(i)
))
for i in range(20):
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=100
)
expect(next_job).not_to_be_null()
expect(next_job['page']).to_equal(str(pages[19 - i].uuid))示例7: test_increases_page_score_when_all_pages_have_been_reviewed
# 需要导入模块: from holmes.models import Page [as 别名]
# 或者: from holmes.models.Page import get_next_job [as 别名]
def test_increases_page_score_when_all_pages_have_been_reviewed(self):
page = PageFactory.create(last_review_date=datetime(2014, 10, 10, 10, 10, 10))
page2 = PageFactory.create(last_review_date=datetime(2014, 10, 10, 10, 10, 10))
settings = Settings.instance(self.db)
settings.lambda_score = 10000
next_job = Page.get_next_job(
self.db,
expiration=100,
cache=self.sync_cache,
lock_expiration=1
)
expect(next_job).to_be_null()
self.db.refresh(page)
self.db.refresh(page2)
expect(page.score).to_equal(5000)
expect(page2.score).to_equal(5000)示例8: _load_next_job
# 需要导入模块: from holmes.models import Page [as 别名]
# 或者: from holmes.models.Page import get_next_job [as 别名]
def _load_next_job(self):
return Page.get_next_job(
self.db,
self.config.REVIEW_EXPIRATION_IN_SECONDS,
self.cache,
self.config.NEXT_JOB_URL_LOCK_EXPIRATION_IN_SECONDS)