本文整理匯總了Python中joblib.pool.MemmapingPool.imap_unordered方法的典型用法代碼示例。如果您正苦於以下問題：Python MemmapingPool.imap_unordered方法的具體用法？Python MemmapingPool.imap_unordered怎麽用？Python MemmapingPool.imap_unordered使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類joblib.pool.MemmapingPool的用法示例。
在下文中一共展示了MemmapingPool.imap_unordered方法的1個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: StatefulPool
# 需要導入模塊: from joblib.pool import MemmapingPool [as 別名]
# 或者: from joblib.pool.MemmapingPool import imap_unordered [as 別名]
class StatefulPool(object):
def __init__(self):
self.n_parallel = 1
self.pool = None
self.queue = None
self.worker_queue = None
self.G = SharedGlobal()
def initialize(self, n_parallel):
self.n_parallel = n_parallel
if self.pool is not None:
print("Warning: terminating existing pool")
self.pool.terminate()
self.queue.close()
self.worker_queue.close()
self.G = SharedGlobal()
if n_parallel > 1:
self.queue = mp.Queue()
self.worker_queue = mp.Queue()
self.pool = MemmapingPool(
self.n_parallel,
temp_folder="/tmp",
)
def run_each(self, runner, args_list=None):
"""
Run the method on each worker process, and collect the result of execution.
The runner method will receive 'G' as its first argument, followed by the arguments
in the args_list, if any
:return:
"""
if args_list is None:
args_list = [tuple()] * self.n_parallel
assert len(args_list) == self.n_parallel
if self.n_parallel > 1:
#return [runner(self.G, *args_list[i]) for i in range(self.n_parallel)]
results = self.pool.map_async(
_worker_run_each, [(runner, args) for args in args_list]
)
for i in range(self.n_parallel):
self.worker_queue.get()
for i in range(self.n_parallel):
self.queue.put(None)
return results.get()
return [runner(self.G, *args_list[0])]
def run_map(self, runner, args_list):
if self.n_parallel > 1:
return self.pool.map(_worker_run_map, [(runner, args) for args in args_list])
else:
ret = []
for args in args_list:
ret.append(runner(self.G, *args))
return ret
def run_imap_unordered(self, runner, args_list):
if self.n_parallel > 1:
for x in self.pool.imap_unordered(_worker_run_map, [(runner, args) for args in args_list]):
yield x
else:
for args in args_list:
yield runner(self.G, *args)
def run_collect(self, collect_once, threshold, args=None, show_prog_bar=True, multi_task=False):
"""
Run the collector method using the worker pool. The collect_once method will receive 'G' as
its first argument, followed by the provided args, if any. The method should return a pair of values.
The first should be the object to be collected, and the second is the increment to be added.
This will continue until the total increment reaches or exceeds the given threshold.
Sample script:
def collect_once(G):
return 'a', 1
stateful_pool.run_collect(collect_once, threshold=3) # => ['a', 'a', 'a']
:param collector:
:param threshold:
:return:
"""
if args is None:
args = tuple()
if self.pool and multi_task:
manager = mp.Manager()
counter = manager.Value('i', 0)
lock = manager.RLock()
inputs = [(collect_once, counter, lock, threshold, arg) for arg in args]
results = self.pool.map_async(
_worker_run_collect,
inputs,
)
if show_prog_bar:
pbar = ProgBarCounter(threshold)
last_value = 0
while True:
time.sleep(0.1)
with lock:
if counter.value >= threshold:
#.........這裏部分代碼省略.........