本文整理匯總了Python中spider.Spider.get_key_pattern方法的典型用法代碼示例。如果您正苦於以下問題：Python Spider.get_key_pattern方法的具體用法？Python Spider.get_key_pattern怎麽用？Python Spider.get_key_pattern使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類spider.Spider的用法示例。
在下文中一共展示了Spider.get_key_pattern方法的1個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: SpiderTest
# 需要導入模塊: from spider import Spider [as 別名]
# 或者: from spider.Spider import get_key_pattern [as 別名]
#.........這裏部分代碼省略.........
'<a href="http://a/b/c"></a>',
'<a href="a/b/c"></a>',
'<a href="/a/b/c"></a>',
'<a href="a"></a>',
'<a href="#"></a>',
'<a href="javascript"></a>',
'</body></html>'))
l1 = ['http://a/b/c', 'a/b/c', '/a/b/c', 'a', '#',
'javascript']
l2 = self.spider.get_all_links(content)
self.assertEqual(l1, l2)
def test_convert_to_unicode_gbk(self):
s = u'測試'
sgbk = s.encode('gbk')
r = self.spider.convert_to_unicode(sgbk)
self.assertEqual(r, s)
def test_convert_to_unicode_gb2312(self):
s = u'測試'
sgb2312 = s.encode('gb2312')
r = self.spider.convert_to_unicode(sgb2312)
self.assertEqual(r, s)
def test_convert_to_unicode_utf8(self):
s = u'測試'
sutf8 = s.encode('utf-8')
r = self.spider.convert_to_unicode(sutf8)
self.assertEqual(r, s)
def test_convert_to_unicode_unknown(self):
s = u'測試'
sunknown = s.encode('utf-16')
self.assertRaises(Exception, self.spider.convert_to_unicode,
sunknown)
def test_filter_links_with_fragment(self):
links = ['http://a#one', 'https://b/c/d#one#two']
flinks = self.spider.filter_links(links)
flinks.sort()
links = ['http://a', 'https://b/c/d']
links.sort()
self.assertEqual(flinks, links)
def test_filter_links_with_relative_url(self):
links = ['www.google.com']
flinks = self.spider.filter_links(links)
links = ['http://www.google.com']
self.assertEqual(flinks, links)
def test_filter_links_with_duplicated_url(self):
links = ['http://www.google.com', 'http://www.google.com']
flinks = self.spider.filter_links(links)
links = ['http://www.google.com']
self.assertEqual(flinks, links)
def test_filter_links_with_invalid_url(self):
links = ['http://', 'https://']
flinks = self.spider.filter_links(links)
self.assertEqual(flinks, [])
def test_get_logger(self):
logger = self.spider.get_logger('/tmp/test.log', 5)
logger.debug('logger test')
log_msg_exist = False
with open('/tmp/test.log') as f:
lines = f.readlines()
for line in lines:
if line.strip().endswith('logger test'):
log_msg_exist = not log_msg_exist
os.remove('/tmp/test.log')
self.assertTrue(log_msg_exist)
def test_verify_page_headers(self):
headers = {'ETag': '123',
'Last-Modified': '456',
'Content-Type': 'text/html'}
r = self.spider.verify_page_headers(headers)
self.assertEqual(r.get('etag', ''), '123')
self.assertEqual(r.get('lastmodified', ''), '456')
def test_verify_page_headers_with_invalid_input(self):
headers = {'content-type': 'msdoc'}
self.assertRaises(Exception,
self.spider.verify_page_headers,
headers)
def test_get_key_pattern_with_invalid_input(self):
key = u'測試'.encode('utf-16')
k = self.spider.get_key_pattern(key)
self.assertEqual(k, None)
key = None
k = self.spider.get_key_pattern(key)
self.assertEqual(k, None)
def test_get_pattern(self):
key = u'測試'.encode('gbk')
k = self.spider.get_key_pattern(key)
r = k.search(u'我是測試')
self.assertNotEqual(r, None)