本文整理汇总了Python中holmes.reviewer.Reviewer._wait_for_async_requests方法的典型用法代码示例。如果您正苦于以下问题：Python Reviewer._wait_for_async_requests方法的具体用法？Python Reviewer._wait_for_async_requests怎么用？Python Reviewer._wait_for_async_requests使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类holmes.reviewer.Reviewer的用法示例。
在下文中一共展示了Reviewer._wait_for_async_requests方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_link_looks_like_image
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_link_looks_like_image(self):
page = PageFactory.create(url="http://globo.com/")
reviewer = Reviewer(
api_url="http://localhost:2368",
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[],
)
content = '<html><a href="http://globo.com/metal.png">Metal</a></html>'
result = {"url": page.url, "status": 200, "content": content, "html": lxml.html.fromstring(content)}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.add_fact = Mock()
facter.async_get = Mock()
facter.get_facts()
expect(facter.add_fact.call_args_list).to_include(call(key="page.links", value=set([])))
expect(facter.add_fact.call_args_list).to_include(call(key="total.number.links", value=0))
expect(facter.async_get.called).to_be_false()示例2: test_can_get_fact_definitions
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_can_get_fact_definitions(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = self.get_file('globo.html')
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = TitleFacter(reviewer)
definitions = facter.get_fact_definitions()
expect(definitions).to_length(1)
expect('page.title' in definitions).to_be_true()示例3: test_handle_url_loaded
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_handle_url_loaded(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url="http://localhost:2368",
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[],
)
content = self.get_file("globo.html")
result = {"url": page.url, "status": 200, "content": content, "html": lxml.html.fromstring(content)}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.async_get = Mock()
facter.get_facts()
facter.handle_url_loaded(page.url, response)
expect(facter.review.data).to_include("page.links")
data = set([(page.url, response)])
expect(facter.review.data["page.links"]).to_equal(data)示例4: test_no_title_tag
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_no_title_tag(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = '<html></html>'
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
reviewer.content_loaded(page.url, Mock(status_code=200, text=content, headers={}))
facter = TitleFacter(reviewer)
facter.add_fact = Mock()
facter.get_facts()
expect(facter.add_fact.called).to_be_false()
expect(facter.review.data).to_be_like({})示例5: test_javascript_link
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_javascript_link(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url="http://localhost:2368",
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[],
)
content = "<script>document.write('<A HREF=\"'+OAS_url+'click_nx.ads/'+OAS_sitepage+'/1'+OAS_rns+'@'+OAS_listpos+'!'+pos+'?'+OAS_query+'\" TARGET='+OAS_target+'>');</script>"
result = {"url": page.url, "status": 200, "content": content, "html": lxml.html.fromstring(content)}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.add_fact = Mock()
facter.async_get = Mock()
facter.get_facts()
expect(facter.add_fact.call_args_list).to_include(call(key="page.links", value=set([])))
expect(facter.add_fact.call_args_list).to_include(call(key="total.number.links", value=0))示例6: test_can_get_facts
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_can_get_facts(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = self.get_file('globo.html')
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = MetaTagsFacter(reviewer)
facter.add_fact = Mock()
facter.get_facts()
values = [{'content': 'utf-8', 'property': None, 'key': 'charset'},
{'content': 'text/html;charset=UTF-8', 'property': 'http-equiv', 'key': 'Content-Type'},
{'content': 'BKmmuVQac1JM6sKlj3IoXQvffyIRJvJfbicMouA2a88', 'property': 'name', 'key': 'google-site-verification'},
{'content': 'width=device-width, initial-scale=1.0, maximum-scale=1.0', 'property': 'name', 'key': 'viewport'},
{'content': u'globo.com - Absolutamente tudo sobre not\xedcias, esportes e entretenimento', 'property': 'property', 'key': 'og:title'},
{'content': 'website', 'property': 'property', 'key': 'og:type'},
{'content': 'http://www.globo.com/', 'property': 'property', 'key': 'og:url'},
{'content': 'http://s.glbimg.com/en/ho/static/globocom2012/img/gcom_marca_og.jpg', 'property': 'property', 'key': 'og:image'},
{'content': 'globo.com', 'property': 'property', 'key': 'og:site_name'},
{'content': u'S\xf3 na globo.com voc\xea encontra tudo sobre o conte\xfado e marcas das Organiza\xe7\xf5es Globo. O melhor acervo de v\xeddeos online sobre entretenimento, esportes e jornalismo do Brasil.', 'property': 'property', 'key': 'og:description'},
{'content': '224969370851736', 'property': 'property', 'key': 'fb:page_id'},
{'content': u'S\xf3 na globo.com voc\xea encontra tudo sobre o conte\xfado e marcas das Organiza\xe7\xf5es Globo. O melhor acervo de v\xeddeos online sobre entretenimento, esportes e jornalismo do Brasil.', 'property': 'name', 'key': 'description'},
{'content': u'Not\xedcias, Entretenimento, Esporte, Tecnologia, Portal, Conte\xfado, Rede Globo, TV Globo, V\xeddeos, Televis\xe3o', 'property': 'name', 'key': 'keywords'},
{'content': 'Globo.com', 'property': 'name', 'key': 'application-name'},
{'content': '#0669DE', 'property': 'name', 'key': 'msapplication-TileColor'},
{'content': 'http://s.glbimg.com/en/ho/static/globocom2012/img/globo-win-tile.png', 'property': 'name', 'key': 'msapplication-TileImage'}]
expect(facter.add_fact.call_args_list).to_include(
call(
key='meta.tags',
value=values,
))
expect(facter.review.data).to_length(1)
expect(facter.review.data).to_include('meta.tags')
expect(facter.review.data).to_be_like({'meta.tags': values})示例7: test_invalid_link
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_invalid_link(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = '<html><a href="http://]http://www.globo.com/malhacao">blah</a></html>'
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.add_fact = Mock()
facter.async_get = Mock()
facter.get_facts()
expect(facter.add_fact.call_args_list).to_include(
call(
key='page.links',
value=set([])
))
expect(facter.add_fact.call_args_list).to_include(
call(
key='total.number.links',
value=0
))
expect(facter.add_fact.call_args_list).to_include(
call(
key='total.number.invalid_links',
value=1
))
expect(facter.add_fact.call_args_list).to_include(
call(
key='page.invalid_links',
value=set(['http://]http://www.globo.com/malhacao'])
))示例8: test_no_get_url_that_exceed_max_url_level
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_no_get_url_that_exceed_max_url_level(self):
page = PageFactory.create(url='http://m.com/')
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = (
'<html>'
'<a href="http://m.com/test/">test</a>'
'<a href="http://m.com/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/">m</a>'
'<a href="http://m.com/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/">m</a>'
'</html>'
)
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.add_fact = Mock()
facter.get_facts()
expect(facter.add_fact.call_args_list).to_equal([
call(
key='page.links',
value=set([])
),
call(
key='total.number.links',
value=2
),
call(
key='total.number.invalid_links',
value=0
),
call(
key='page.invalid_links',
value=set([])
)])示例9: test_not_get_links_with_nofollow
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_not_get_links_with_nofollow(self):
page = PageFactory.create(url='http://m.com/')
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = '<html>' \
'<a href="http://m.com/test/">test</a>' \
'<a href="http://m.com/metal/" rel="nofollow">metal</a>' \
'</html>'
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.add_fact = Mock()
facter.get_facts()
expect(facter.add_fact.call_args_list).to_equal([
call(
key='page.links',
value=set([])
),
call(
key='total.number.links',
value=1
),
call(
key='total.number.invalid_links',
value=0
),
call(
key='page.invalid_links',
value=set([])
)])示例10: test_url_ends_with_slash
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_url_ends_with_slash(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = self.get_file('globo.html')
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.add_fact = Mock()
facter.async_get = Mock()
facter.get_facts()
expect(facter.review.data).to_include('page.links')
expect(facter.async_get.call_args_list).to_include(
call(
'http://my-site.com',
facter.handle_url_loaded
))
expect(facter.async_get.call_args_list).to_include(
call(
'http://my-site.com/privacidade.html',
facter.handle_url_loaded
))
expect(facter.async_get.call_args_list).to_include(
call(
'http://my-site.com/todos-os-sites.html',
facter.handle_url_loaded
))示例11: test_ignore_base64_images
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_ignore_base64_images(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = '<html><img src="data:image/png;base64,iVBOR" alt="a" title="b" /></html>'
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = ImageFacter(reviewer)
facter.add_fact = Mock()
facter.async_get = Mock()
facter.get_facts()
expect(facter.add_fact.call_args_list).to_include(
call(
key='page.images',
value=set([]),
))
expect(facter.add_fact.call_args_list).to_include(
call(
key='total.size.img',
value=0,
)),
expect(facter.add_fact.call_args_list).to_include(
call(
key='total.requests.img',
value=0,
))示例12: test_can_get_facts
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_can_get_facts(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url="http://localhost:2368",
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[],
)
content = self.get_file("globo.html")
result = {"url": page.url, "status": 200, "content": content, "html": lxml.html.fromstring(content)}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.add_fact = Mock()
facter.async_get = Mock()
facter.get_facts()
expect(facter.review.data).to_length(2)
expect(facter.review.data).to_include("page.links")
expect(facter.review.data["page.links"]).to_equal(set([]))
expect(facter.review.data).to_include("page.all_links")
link = facter.review.data["page.all_links"][1]
expect(link.tag).to_equal("a")
expect(link.get("href")).to_equal("/")
expect(link.get("title")).to_equal("globo.com")
expect(facter.async_get.call_count).to_equal(335)
expect(facter.add_fact.call_args_list).to_include(call(key="page.links", value=set([])))
expect(facter.add_fact.call_args_list).to_include(call(key="total.number.links", value=489))
expect(facter.add_fact.call_args_list).to_include(call(key="total.number.invalid_links", value=0))
expect(facter.add_fact.call_args_list).to_include(call(key="page.invalid_links", value=set([])))示例13: test_handle_url_loaded
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_handle_url_loaded(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = '<html><img src="test.png" alt="a" title="b" /></html>'
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = ImageFacter(reviewer)
facter.async_get = Mock()
facter.get_facts()
facter.handle_url_loaded(page.url, response)
expect(facter.review.data).to_include('page.all_images')
expect(facter.review.data['page.all_images']).not_to_be_null()
img_src = facter.review.data['page.all_images'][0].get('src')
expect(img_src).to_equal('test.png')
expect(facter.review.data).to_include('page.images')
data = set([(page.url, response)])
expect(facter.review.data['page.images']).to_equal(data)
expect(facter.review.data).to_include('total.size.img')
expect(facter.review.data['total.size.img']).to_equal(0.0517578125)示例14: test_javascript_link
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_javascript_link(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = '<script>document.write(\'<A HREF="\'+OAS_url+\'click_nx.ads/\'+OAS_sitepage+\'/1\'+OAS_rns+\'@\'+OAS_listpos+\'!\'+pos+\'?\'+OAS_query+\'" TARGET=\'+OAS_target+\'>\');</script>'
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = LinkFacter(reviewer)
facter.add_fact = Mock()
facter.async_get = Mock()
facter.get_facts()
expect(facter.add_fact.call_args_list).to_include(
call(
key='page.links',
value=set([])
))
expect(facter.add_fact.call_args_list).to_include(
call(
key='total.number.links',
value=0
))示例15: test_can_get_facts
# 需要导入模块: from holmes.reviewer import Reviewer [as 别名]
# 或者: from holmes.reviewer.Reviewer import _wait_for_async_requests [as 别名]
def test_can_get_facts(self):
page = PageFactory.create()
reviewer = Reviewer(
api_url='http://localhost:2368',
page_uuid=page.uuid,
page_url=page.url,
page_score=0.0,
config=Config(),
facters=[]
)
content = self.get_file('globo.html')
result = {
'url': page.url,
'status': 200,
'content': content,
'html': lxml.html.fromstring(content)
}
reviewer.responses[page.url] = result
reviewer._wait_for_async_requests = Mock()
reviewer.save_review = Mock()
response = Mock(status_code=200, text=content, headers={})
reviewer.content_loaded(page.url, response)
facter = GoogleAnalyticsFacter(reviewer)
facter.add_fact = Mock()
facter.async_get = Mock()
facter.get_facts()
expect(facter.review.data).to_length(1)
expect(facter.review.data).to_include('page.google_analytics')
expect(facter.add_fact.call_args_list).to_include(
call(
key='page.google_analytics',
value=set([
('UA-296593-2', 'www.globo.com'),
('UA-296593-15', '.globo.com')
])
))