本文整理汇总了Python中gevent.pool.Group方法的典型用法代码示例。如果您正苦于以下问题:Python pool.Group方法的具体用法?Python pool.Group怎么用?Python pool.Group使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类gevent.pool
的用法示例。
在下文中一共展示了pool.Group方法的10个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: forward
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def forward(self,sock,remote_sock,data_timeout=5*60):
u"""在两个套接字之间转发数据(阻塞调用)
在转发失败时自动关闭连接。在双向都出现超时的情况下会关闭连接。
未专门处理 shutdown ,单方向 shutdown 时会关闭双向链接。
"""
try:
o = {
# 最后一次转发数据的时间 = int(time()*1000)
'forward_data_time':int(time.time()*1000),
}
sock.settimeout(data_timeout)
remote_sock.settimeout(data_timeout)
group = Group()
group.add(gevent.spawn(self.__forwardData,sock,remote_sock,o,data_timeout))
group.add(gevent.spawn(self.__forwardData,remote_sock,sock,o,data_timeout))
group.join()
finally:
sock.close()
remote_sock.close()
示例2: __init__
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def __init__(self, nsqd_tcp_addresses=[], max_backoff_duration=128,
**kwargs):
if not nsqd_tcp_addresses:
raise ValueError('must specify at least one nsqd or lookupd')
self.nsqd_tcp_addresses = parse_nsqds(nsqd_tcp_addresses)
self.max_backoff_duration = max_backoff_duration
self.conn_kwargs = kwargs
self.logger = logging.getLogger(__name__)
self._state = INIT
self._connections = Queue()
self._connection_backoffs = defaultdict(self._create_backoff)
self._response_queues = {}
self._workers = Group()
示例3: spawn_later
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def spawn_later(self, seconds, func, *args, **kwargs):
parent = super(GreenletGroupWithExceptionCatching, self)
func_wrap = self._wrap_errors(func)
# spawn_later doesn't exist in pool.Group, so let's implement it below
greenlet = parent.greenlet_class(func_wrap, *args, **kwargs)
parent.add(greenlet)
greenlet.start_later(seconds)
return greenlet
示例4: spawn_imap_unordered
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def spawn_imap_unordered(self):
igroup = Group()
result = []
with tracer.start_active_span('test'):
for i in igroup.imap_unordered(self.make_http_call, range(3)):
result.append(i)
示例5: parallel_map
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def parallel_map(func, iterable, args=None, kwargs=None, workers=None):
"""Map func on a list using gevent greenlets.
:param func: function applied on iterable elements
:type func: function
:param iterable: elements to map the function over
:type iterable: iterable
:param args: arguments of func
:type args: tuple
:param kwargs: keyword arguments of func
:type kwargs: dict
:param workers: limit the number of greenlets
running in parrallel
:type workers: int
"""
if args is None:
args = ()
if kwargs is None:
kwargs = {}
if workers is not None:
pool = Pool(workers)
else:
pool = Group()
iterable = [pool.spawn(func, i, *args, **kwargs) for i in iterable]
pool.join(raise_error=True)
for idx, i in enumerate(iterable):
i_type = type(i.get())
i_value = i.get()
if issubclass(i_type, BaseException):
raise i_value
iterable[idx] = i_value
return iterable
示例6: _filter_transactions
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def _filter_transactions(self, transactions: List[Transaction]) -> List[Transaction]:
"""Filters all transactions in parallel. The results may not have the same order."""
filter_task_group = pool.Group()
filter_task_results = filter_task_group.imap_unordered(self._filter_transaction_task, transactions)
return filter_array(lambda el: el is not None, filter_task_results)
示例7: __init__
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def __init__(self, account,
heartbeat=1, refresh_frequency=30):
self.refresh_frequency = refresh_frequency
self.syncmanager_lock = BoundedSemaphore(1)
self.saved_remote_folders = None
self.sync_engine_class = FolderSyncEngine
self.folder_monitors = Group()
self.delete_handler = None
BaseMailSyncMonitor.__init__(self, account, heartbeat)
示例8: __init__
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def __init__(self, topic, channel, nsqd_tcp_addresses=[],
lookupd_http_addresses=[], name=None, message_handler=None,
max_tries=5, max_in_flight=1, requeue_delay=0,
lookupd_poll_interval=60, lookupd_poll_jitter=0.3,
low_ready_idle_timeout=10, max_backoff_duration=128,
backoff_on_requeue=True, **kwargs):
if not nsqd_tcp_addresses and not lookupd_http_addresses:
raise ValueError('must specify at least one nsqd or lookupd')
self.nsqd_tcp_addresses = parse_nsqds(nsqd_tcp_addresses)
self.lookupds = parse_lookupds(lookupd_http_addresses)
self.iterlookupds = cycle(self.lookupds)
self.topic = topic
self.channel = channel
self.max_tries = max_tries
self.max_in_flight = max_in_flight
self.requeue_delay = requeue_delay
self.lookupd_poll_interval = lookupd_poll_interval
self.lookupd_poll_jitter = lookupd_poll_jitter
self.low_ready_idle_timeout = low_ready_idle_timeout
self.backoff_on_requeue = backoff_on_requeue
self.max_backoff_duration = max_backoff_duration
self.conn_kwargs = kwargs
if name:
self.name = name
else:
self.name = '%s.%s.%s' % (__name__, self.topic, self.channel)
if message_handler is not None:
self.on_message.connect(message_handler, weak=False)
self.logger = logging.getLogger(self.name)
self._state = INIT
self._redistributed_ready_event = Event()
self._connection_backoffs = defaultdict(self._create_backoff)
self._message_backoffs = defaultdict(self._create_backoff)
self._connections = {}
self._workers = Group()
self._killables = Group()
示例9: create_connection
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def create_connection(self, address, timeout=10):
ip_list = dnslib.dnsQuery(address[0])
# 尝试连接缓存
route_list = self.get_route_order_ping(address[0],address[1],None)
if route_list:
try:
route = route_list[0]
hit_ip = route['hit_ip']
if hit_ip in ip_list:
cache_timeout = route['tcp_ping']
if cache_timeout<1000:
cache_timeout = cache_timeout * 2
else:
cache_timeout = cache_timeout+1000
cache_timeout = int(math.ceil(cache_timeout/1000.0))
start_time = int(time.time() * 1000)
sock = self._direct_create_connection(address,hit_ip, cache_timeout)
t = int(time.time() * 1000) - start_time
logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s 命中。time:%s'%(self.get_display_name(),address[0],hit_ip,address[1],t))
self.update_route_ping(address[0],address[1],t,hit_ip)
return sock
else:
logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s IP 不匹配,放弃缓存。'%(self.get_display_name(),address[0],hit_ip,address[1]))
except:
t = int(time.time() * 1000) - start_time
info = traceback.format_exc()
logging.debug(
u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s 失败。time:%s' % (self.get_display_name(), address[0],hit_ip, address[1],t))
logging.debug('%s\r\n\r\n' % info)
# 缓存失败,连接全部
evt = Event()
group = Group()
aync_task = DirectAsyncTask(evt, None, group)
for ip in ip_list:
group.add(gevent.spawn(self._create_connection, aync_task, address,ip, timeout))
# 所有连接失败时发出通知
gevent.spawn(self._create_connection_all_end, aync_task)
evt.wait()
if aync_task.sock:
return aync_task.sock
else:
raise UpstreamConnectError()
示例10: create_connection
# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Group [as 别名]
def create_connection(self, address, timeout=10):
# 尝试连接缓存
route_list = self.get_route_order_ping(address[0],address[1],None)
if route_list:
try:
route = route_list[0]
cache_timeout = route['tcp_ping']
if cache_timeout<1000:
cache_timeout = cache_timeout * 2
else:
cache_timeout = cache_timeout+1000
cache_timeout = int(math.ceil(cache_timeout/1000.0))
_upstream = self.upstream_dict.get(route['proxy_name'])
start_time = int(time.time() * 1000)
sock = _upstream.create_connection(address, cache_timeout)
t = int(time.time() * 1000) - start_time
logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s:%s 命中。time:%s'%(_upstream.get_display_name(),address[0],address[1],t))
self.update_route_ping(_upstream.get_name(),address[0],address[1],t)
return sock
except:
t = int(time.time() * 1000) - start_time
info = traceback.format_exc()
logging.debug(
u'[upstream][RouteCache]%s 缓存记录 连接 %s:%s 失败。time:%s' % (_upstream.get_display_name(), address[0], address[1],t))
logging.debug('%s\r\n\r\n' % info)
# 缓存失败,连接全部
evt = Event()
group = Group()
aync_task = MultipathAsyncTask(evt, None, group)
for _upstream in self.upstream_dict.values():
group.add(gevent.spawn(self._create_connection, _upstream, aync_task, address, timeout))
# 所有连接失败时发出通知
gevent.spawn(self._create_connection_all_end, aync_task)
evt.wait()
if aync_task.sock:
return aync_task.sock
else:
raise UpstreamConnectError()