本文整理汇总了Python中grab.proxylist.ProxyList.get_next_proxy方法的典型用法代码示例。如果您正苦于以下问题：Python ProxyList.get_next_proxy方法的具体用法？Python ProxyList.get_next_proxy怎么用？Python ProxyList.get_next_proxy使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类grab.proxylist.ProxyList的用法示例。
在下文中一共展示了ProxyList.get_next_proxy方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_get_next_proxy
# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_next_proxy [as 别名]
def test_get_next_proxy(self):
pl = ProxyList()
path = self.generate_plist_file('foo:1\nbar:1')
pl.load_file(path)
self.assertEqual(pl.get_next_proxy().host, 'foo')
self.assertEqual(pl.get_next_proxy().host, 'bar')
self.assertEqual(pl.get_next_proxy().host, 'foo')
pl.load_file(path)
self.assertEqual(pl.get_next_proxy().host, 'foo')示例2: test_get_next_proxy
# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_next_proxy [as 别名]
def test_get_next_proxy(self):
with temp_file() as path:
plist = ProxyList()
self.generate_plist_file(path, 'foo:1\nbar:1')
plist.load_file(path)
self.assertEqual(plist.get_next_proxy().host, 'foo')
self.assertEqual(plist.get_next_proxy().host, 'bar')
self.assertEqual(plist.get_next_proxy().host, 'foo')
plist.load_file(path)
self.assertEqual(plist.get_next_proxy().host, 'foo')示例3: Grab
# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_next_proxy [as 别名]
#.........这里部分代码省略.........
"""
self.reset()
if isinstance(content, six.text_type):
raise error.GrabMisuseError('Method `setup_document` accepts only '
'byte string in `content` argument.')
# Configure Document instance
doc = Document(grab=self)
doc.body = content
doc.status = ''
doc.head = b'HTTP/1.1 200 OK\r\n\r\n'
doc.parse(charset=kwargs.get('document_charset'))
doc.code = 200
doc.total_time = 0
doc.connect_time = 0
doc.name_lookup_time = 0
doc.url = ''
for key, value in kwargs.items():
setattr(doc, key, value)
self.doc = doc
def change_proxy(self, random=True):
"""
Set random proxy from proxylist.
"""
if self.proxylist.size():
if random:
proxy = self.proxylist.get_random_proxy()
else:
proxy = self.proxylist.get_next_proxy()
self.setup(proxy=proxy.get_address(),
proxy_userpwd=proxy.get_userpwd(),
proxy_type=proxy.proxy_type)
else:
logger.debug('Proxy list is empty')
#
# Private methods
#
@classmethod
def common_headers(cls):
"""
Build headers which sends typical browser.
"""
return {
'Accept': 'text/xml,application/xml,application/xhtml+xml'
',text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.%d'
% randint(2, 5),
'Accept-Language': 'en-us,en;q=0.%d' % (randint(5, 9)),
'Accept-Charset': 'utf-8,windows-1251;q=0.7,*;q=0.%d'
% randint(5, 7),
'Keep-Alive': '300',
}
def save_dumps(self):
# pylint: disable=no-member
thread_name = threading.currentThread().getName().lower()
# pylint: enable=no-member
if thread_name == 'mainthread':
thread_name = ''示例4: Grab
# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_next_proxy [as 别名]
#.........这里部分代码省略.........
doc.status = ''
doc.head = ''
doc.parse(charset=kwargs.get('document_charset'))
doc.code = 200
doc.total_time = 0
doc.connect_time = 0
doc.name_lookup_time = 0
doc.url = ''
for key, value in kwargs.items():
setattr(doc, key, value)
self.doc = doc
def change_proxy(self):
"""
Set random proxy from proxylist.
"""
if self.proxylist.size():
proxy = self.proxylist.get_random_proxy()
self.setup(proxy=proxy.get_address(),
proxy_userpwd=proxy.get_userpwd(),
proxy_type=proxy.proxy_type)
else:
logger.debug('Proxy list is empty')
def use_next_proxy(self):
"""
Set next proxy from proxylist.
"""
if self.proxylist.size():
proxy = self.proxylist.get_next_proxy()
self.setup(proxy=proxy.get_address(),
proxy_userpwd=proxy.get_userpwd(),
proxy_type=proxy.proxy_type)
else:
logger.debug('Proxy list is empty')
"""
Private methods
"""
def common_headers(self):
"""
Build headers which sends typical browser.
"""
return {
'Accept': 'text/xml,application/xml,application/xhtml+xml'
',text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.%d'
% randint(2, 5),
'Accept-Language': 'en-us,en;q=0.%d' % (randint(5, 9)),
'Accept-Charset': 'utf-8,windows-1251;q=0.7,*;q=0.%d'
% randint(5, 7),
'Keep-Alive': '300',
'Expect': '',
}
def save_dumps(self):
if self.config['log_dir']:
thread_name = threading.currentThread().getName().lower()
if thread_name == 'mainthread':
thread_name = ''
else: