本文整理汇总了Python中grab.proxylist.ProxyList.get_random方法的典型用法代码示例。如果您正苦于以下问题：Python ProxyList.get_random方法的具体用法？Python ProxyList.get_random怎么用？Python ProxyList.get_random使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类grab.proxylist.ProxyList的用法示例。
在下文中一共展示了ProxyList.get_random方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: Spider
# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random [as 别名]
#.........这里部分代码省略.........
logger.debug('Got StopTaskProcessing exception')
stop = True
break
else:
if isinstance(mid_response, Task):
logger.debug('Got task from middleware')
self.add_task(mid_response)
stop = True
break
elif mid_response is None:
pass
else:
raise Exception('Unknown response from middleware %s' % mid)
# TOFIX: end
if stop:
return
if self.ng:
logger_verbose.debug('Submitting result for task %s to response'
' queue' % res['task'])
self.network_response_queue.put(res)
else:
handler = self.find_task_handler(res['task'])
self.execute_task_handler(res, handler)
def change_proxy(self, task, grab):
"""
Assign new proxy from proxylist to the task.
"""
if task.use_proxylist and self.proxylist_enabled:
if self.proxy_auto_change:
self.proxy = self.proxylist.get_random()
if self.proxy:
proxy, proxy_userpwd, proxy_type = self.proxy
grab.setup(proxy=proxy, proxy_userpwd=proxy_userpwd,
proxy_type=proxy_type)
def process_new_task(self, task):
"""
Handle new task.
1) Setup Grab object for that task
2) Try to load task from the cache
3) If no cached data then submit task to network transport
"""
grab = self.setup_grab_for_task(task)
grab_config_backup = grab.dump_config()
cache_result = None
if self.is_task_cacheable(task, grab):
with self.save_timer('cache'):
with self.save_timer('cache.read'):
cache_result = self.load_task_from_cache(
self.transport, task, grab, grab_config_backup)
if cache_result:
logger_verbose.debug('Task data is loaded from the cache. '
'Yielding task result.')
self.process_network_result(cache_result, from_cache=True)
self.inc_count('task-%s-cache' % task.name)
else:
if self.only_cache:
logger.debug('Skipping network request to %s' % grab.config['url'])示例2: Spider
# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random [as 别名]
#.........这里部分代码省略.........
self.inc_count('request-processed')
self.inc_count('task')
self.inc_count('task-%s' % res['task'].name)
if (res['task'].network_try_count == 1 and
res['task'].task_try_count == 1):
self.inc_count('task-%s-initial' % res['task'].name)
# Update traffic statistics
if res['grab'] and res['grab'].response:
self.timers['network-name-lookup'] +=\
res['grab'].response.name_lookup_time
self.timers['network-connect'] += res['grab'].response.connect_time
self.timers['network-total'] += res['grab'].response.total_time
if not from_cache:
self.inc_count('download-size',
res['grab'].response.download_size)
self.inc_count('upload-size',
res['grab'].response.upload_size)
self.inc_count('download-size-with-cache',
res['grab'].response.download_size)
self.inc_count('upload-size-with-cache',
res['grab'].response.upload_size)
handler = self.find_task_handler(res['task'])
self.execute_task_handler(res, handler)
def change_proxy(self, task, grab):
"""
Assign new proxy from proxylist to the task.
"""
if task.use_proxylist and self.proxylist_enabled:
if self.proxy_auto_change:
self.proxy = self.proxylist.get_random()
if self.proxy:
proxy, proxy_userpwd, proxy_type = self.proxy
grab.setup(proxy=proxy, proxy_userpwd=proxy_userpwd,
proxy_type=proxy_type)
def process_new_task(self, task):
"""
Handle new task.
1) Setup Grab object for that task
2) Try to load task from the cache
3) If no cached data then submit task to network transport
"""
grab = self.setup_grab_for_task(task)
grab_config_backup = grab.dump_config()
cache_result = None
if self.is_task_cacheable(task, grab):
with self.save_timer('cache'):
with self.save_timer('cache.read'):
cache_result = self.load_task_from_cache(
self.transport, task, grab, grab_config_backup)
if cache_result:
logger_verbose.debug('Task data is loaded from the cache. '
'Yielding task result.')
self.process_network_result(cache_result, from_cache=True)
self.inc_count('task-%s-cache' % task.name)
else:
if self.only_cache:
logger.debug('Skipping network request to %s' %