本文整理汇总了Python中spider.Spider.get_all_links方法的典型用法代码示例。如果您正苦于以下问题：Python Spider.get_all_links方法的具体用法？Python Spider.get_all_links怎么用？Python Spider.get_all_links使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类spider.Spider的用法示例。
在下文中一共展示了Spider.get_all_links方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: SpiderTest
# 需要导入模块: from spider import Spider [as 别名]
# 或者: from spider.Spider import get_all_links [as 别名]
class SpiderTest(unittest.TestCase):
def setUp(self):
self.spider = Spider('http://www.google.com',
loglevel=1)
def tearDown(self):
self.spider.logger.handlers = []
def test_is_valid_url_with_valid_url(self):
r = self.spider.is_valid_url('http://www.google.com')
self.assertTrue(r)
r = self.spider.is_valid_url('https://www.google.com')
self.assertTrue(r)
def test_is_valid_url_with_invalid_url(self):
r = self.spider.is_valid_url('www.google.com')
self.assertFalse(r)
r = self.spider.is_valid_url('google')
self.assertFalse(r)
def test_get_abs_url_with_valid_base(self):
r = self.spider.get_abs_url(r'http://www.google.com/a/', 'b')
self.assertEqual(r, r'http://www.google.com/a/b')
r = self.spider.get_abs_url(r'http://www.google.com/a', 'b')
self.assertEqual(r, r'http://www.google.com/b')
r = self.spider.get_abs_url(r'https://www.google.com/a/c', '/b')
self.assertEqual(r, r'https://www.google.com/b')
r = self.spider.get_abs_url(r'http://www.google.com',
r'http://www.facebook.com')
self.assertEqual(r, r'http://www.facebook.com')
r = self.spider.get_abs_url(None, r'http://www.google.com')
self.assertEqual(r, r'http://www.google.com')
def test_get_abs_url_with_invalid_base(self):
r = self.spider.get_abs_url('http://', 'www.google.com')
self.assertEqual(r, 'http://www.google.com')
def test_get_abs_url_with_invalid_url(self):
r = self.spider.get_abs_url(None, 'http://')
self.assertEqual(r, 'http://')
def test_get_all_links_with_valid_content(self):
content = ''.join(('<html><body>',
'<a href="http://a/b/c"></a>',
'<a href="a/b/c"></a>',
'<a href="/a/b/c"></a>',
'<a href="a"></a>',
'<a href="#"></a>',
'<a href="javascript"></a>',
'</body></html>'))
l1 = ['http://a/b/c', 'a/b/c', '/a/b/c', 'a', '#',
'javascript']
l2 = self.spider.get_all_links(content)
self.assertEqual(l1, l2)
def test_convert_to_unicode_gbk(self):
s = u'测试'
sgbk = s.encode('gbk')
r = self.spider.convert_to_unicode(sgbk)
self.assertEqual(r, s)
def test_convert_to_unicode_gb2312(self):
s = u'测试'
sgb2312 = s.encode('gb2312')
r = self.spider.convert_to_unicode(sgb2312)
self.assertEqual(r, s)
def test_convert_to_unicode_utf8(self):
s = u'测试'
sutf8 = s.encode('utf-8')
r = self.spider.convert_to_unicode(sutf8)
self.assertEqual(r, s)
def test_convert_to_unicode_unknown(self):
s = u'测试'
sunknown = s.encode('utf-16')
self.assertRaises(Exception, self.spider.convert_to_unicode,
sunknown)
def test_filter_links_with_fragment(self):
links = ['http://a#one', 'https://b/c/d#one#two']
flinks = self.spider.filter_links(links)
flinks.sort()
links = ['http://a', 'https://b/c/d']
links.sort()
self.assertEqual(flinks, links)
def test_filter_links_with_relative_url(self):
links = ['www.google.com']
flinks = self.spider.filter_links(links)
links = ['http://www.google.com']
self.assertEqual(flinks, links)
def test_filter_links_with_duplicated_url(self):
links = ['http://www.google.com', 'http://www.google.com']
flinks = self.spider.filter_links(links)
links = ['http://www.google.com']
self.assertEqual(flinks, links)
def test_filter_links_with_invalid_url(self):
#.........这里部分代码省略.........