本文整理匯總了Python中spider.Spider.is_valid_url方法的典型用法代碼示例。如果您正苦於以下問題：Python Spider.is_valid_url方法的具體用法？Python Spider.is_valid_url怎麽用？Python Spider.is_valid_url使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類spider.Spider的用法示例。
在下文中一共展示了Spider.is_valid_url方法的1個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: SpiderTest
# 需要導入模塊: from spider import Spider [as 別名]
# 或者: from spider.Spider import is_valid_url [as 別名]
class SpiderTest(unittest.TestCase):
def setUp(self):
self.spider = Spider('http://www.google.com',
loglevel=1)
def tearDown(self):
self.spider.logger.handlers = []
def test_is_valid_url_with_valid_url(self):
r = self.spider.is_valid_url('http://www.google.com')
self.assertTrue(r)
r = self.spider.is_valid_url('https://www.google.com')
self.assertTrue(r)
def test_is_valid_url_with_invalid_url(self):
r = self.spider.is_valid_url('www.google.com')
self.assertFalse(r)
r = self.spider.is_valid_url('google')
self.assertFalse(r)
def test_get_abs_url_with_valid_base(self):
r = self.spider.get_abs_url(r'http://www.google.com/a/', 'b')
self.assertEqual(r, r'http://www.google.com/a/b')
r = self.spider.get_abs_url(r'http://www.google.com/a', 'b')
self.assertEqual(r, r'http://www.google.com/b')
r = self.spider.get_abs_url(r'https://www.google.com/a/c', '/b')
self.assertEqual(r, r'https://www.google.com/b')
r = self.spider.get_abs_url(r'http://www.google.com',
r'http://www.facebook.com')
self.assertEqual(r, r'http://www.facebook.com')
r = self.spider.get_abs_url(None, r'http://www.google.com')
self.assertEqual(r, r'http://www.google.com')
def test_get_abs_url_with_invalid_base(self):
r = self.spider.get_abs_url('http://', 'www.google.com')
self.assertEqual(r, 'http://www.google.com')
def test_get_abs_url_with_invalid_url(self):
r = self.spider.get_abs_url(None, 'http://')
self.assertEqual(r, 'http://')
def test_get_all_links_with_valid_content(self):
content = ''.join(('<html><body>',
'<a href="http://a/b/c"></a>',
'<a href="a/b/c"></a>',
'<a href="/a/b/c"></a>',
'<a href="a"></a>',
'<a href="#"></a>',
'<a href="javascript"></a>',
'</body></html>'))
l1 = ['http://a/b/c', 'a/b/c', '/a/b/c', 'a', '#',
'javascript']
l2 = self.spider.get_all_links(content)
self.assertEqual(l1, l2)
def test_convert_to_unicode_gbk(self):
s = u'測試'
sgbk = s.encode('gbk')
r = self.spider.convert_to_unicode(sgbk)
self.assertEqual(r, s)
def test_convert_to_unicode_gb2312(self):
s = u'測試'
sgb2312 = s.encode('gb2312')
r = self.spider.convert_to_unicode(sgb2312)
self.assertEqual(r, s)
def test_convert_to_unicode_utf8(self):
s = u'測試'
sutf8 = s.encode('utf-8')
r = self.spider.convert_to_unicode(sutf8)
self.assertEqual(r, s)
def test_convert_to_unicode_unknown(self):
s = u'測試'
sunknown = s.encode('utf-16')
self.assertRaises(Exception, self.spider.convert_to_unicode,
sunknown)
def test_filter_links_with_fragment(self):
links = ['http://a#one', 'https://b/c/d#one#two']
flinks = self.spider.filter_links(links)
flinks.sort()
links = ['http://a', 'https://b/c/d']
links.sort()
self.assertEqual(flinks, links)
def test_filter_links_with_relative_url(self):
links = ['www.google.com']
flinks = self.spider.filter_links(links)
links = ['http://www.google.com']
self.assertEqual(flinks, links)
def test_filter_links_with_duplicated_url(self):
links = ['http://www.google.com', 'http://www.google.com']
flinks = self.spider.filter_links(links)
links = ['http://www.google.com']
self.assertEqual(flinks, links)
def test_filter_links_with_invalid_url(self):
#.........這裏部分代碼省略.........