本文整理匯總了Python中multiprocessing.Semaphore方法的典型用法代碼示例。如果您正苦於以下問題:Python multiprocessing.Semaphore方法的具體用法?Python multiprocessing.Semaphore怎麽用?Python multiprocessing.Semaphore使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類multiprocessing
的用法示例。
在下文中一共展示了multiprocessing.Semaphore方法的15個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: initialize_fallback_semaphores
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def initialize_fallback_semaphores():
"""This needs to be called once at the very beginning of starting ACE."""
# we need some fallback functionality for when the network semaphore server is down
# these semaphores serve that purpose
global_engine_instance_count = saq.CONFIG['global'].getint('global_engine_instance_count')
for key in saq.CONFIG['network_semaphore'].keys():
if key.startswith('semaphore_'):
semaphore_name = key[len('semaphore_'):]
# the configuration settings for the network semaphore specify how many connections
# are allowed to a specific resource at once, globally
# so if we unable to coordinate globally, the fall back is to divide the available
# number of resources between all the engines evenly
# that's what this next equation is for
fallback_limit = int(floor(saq.CONFIG['network_semaphore'].getfloat(key) / float(global_engine_instance_count)))
# we allow a minimum of one per engine
if fallback_limit < 1:
fallback_limit = 1
logging.debug("fallback semaphore count for {0} is {1}".format(semaphore_name, fallback_limit))
fallback_semaphores[semaphore_name] = LoggingSemaphore(fallback_limit)
#fallback_semaphores[semaphore_name] = multiprocessing.Semaphore(fallback_limit)
#fallback_semaphores[semaphore_name].semaphore_name = 'fallback {0}'.format(semaphore_name)
示例2: test_timeout
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_timeout(self):
if self.TYPE != 'processes':
self.skipTest('test not appropriate for {}'.format(self.TYPE))
sem = self.Semaphore(0)
acquire = TimingWrapper(sem.acquire)
self.assertEqual(acquire(False), False)
self.assertTimingAlmostEqual(acquire.elapsed, 0.0)
self.assertEqual(acquire(False, None), False)
self.assertTimingAlmostEqual(acquire.elapsed, 0.0)
self.assertEqual(acquire(False, TIMEOUT1), False)
self.assertTimingAlmostEqual(acquire.elapsed, 0)
self.assertEqual(acquire(True, TIMEOUT2), False)
self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT2)
self.assertEqual(acquire(timeout=TIMEOUT3), False)
self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT3)
示例3: test_waitfor_timeout
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_waitfor_timeout(self):
# based on test in test/lock_tests.py
cond = self.Condition()
state = self.Value('i', 0)
success = self.Value('i', False)
sem = self.Semaphore(0)
p = self.Process(target=self._test_waitfor_timeout_f,
args=(cond, state, success, sem))
p.daemon = True
p.start()
self.assertTrue(sem.acquire(timeout=10))
# Only increment 3 times, so state == 4 is never reached.
for i in range(3):
time.sleep(0.01)
with cond:
state.value += 1
cond.notify()
p.join(5)
self.assertTrue(success.value)
示例4: build_ctrl
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def build_ctrl(self, world_size):
"""
Builds several parallel communication mechanisms for controlling the
workflow across processes.
"""
opt_throttle = (mp.Barrier(world_size) if world_size > 1 else
None)
return AttrDict(
quit=mp.Value('b', lock=True),
quit_opt=mp.RawValue('b'),
sample_ready=[mp.Semaphore(0) for _ in range(2)], # Double buffer.
sample_copied=[mp.Semaphore(1) for _ in range(2)],
sampler_itr=mp.Value('l', lock=True),
opt_throttle=opt_throttle,
eval_time=mp.Value('d', lock=True),
)
示例5: test_get_size
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_get_size(self):
"""
Creates a "producer" process that enqueues ``n`` elements
every ``interval`` seconds. Asserts that a ``get(n, timeout=n*interval+delta)``
yields all ``n`` elements.
"""
mp_queue = mp.Queue()
request_queue = MultiprocessingRequestQueue(mp_queue)
n = 10
interval = 0.1
sem = mp.Semaphore(0)
p = mp.Process(target=_enqueue_on_interval, args=(mp_queue, n, interval, sem))
p.start()
sem.acquire() # blocks until the process has started to run the function
timeout = interval * (n + 1)
start = time.time()
requests = request_queue.get(n, timeout=timeout)
self.assertLessEqual(time.time() - start, timeout + interval)
self.assertEqual(n, len(requests))
示例6: test_get_less_than_size
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_get_less_than_size(self):
"""
Tests slow producer.
Creates a "producer" process that enqueues ``n`` elements
every ``interval`` seconds. Asserts that a ``get(n, timeout=(interval * n/2))``
yields at most ``n/2`` elements.
"""
mp_queue = mp.Queue()
request_queue = MultiprocessingRequestQueue(mp_queue)
n = 10
interval = 0.1
sem = mp.Semaphore(0)
p = mp.Process(target=_enqueue_on_interval, args=(mp_queue, n, interval, sem))
p.start()
sem.acquire() # blocks until the process has started to run the function
requests = request_queue.get(n, timeout=(interval * (n / 2)))
self.assertLessEqual(n / 2, len(requests))
示例7: _request
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def _request(cls, resource):
"""ResourceManager, we recommand using scheduler instead of creating your own
resource manager.
"""
assert cls.check_possible(resource), \
'Requested num_cpu={} and num_gpu={} should be less than or equal to' + \
'largest node availability CPUs={}, GPUs={}'. \
format(resource.num_cpus, resource.num_gpus, cls.MAX_GPU_COUNT, cls.MAX_CPU_COUNT)
with cls.LOCK:
node = cls.check_availability(resource)
if node is not None:
cls.NODE_RESOURCE_MANAGER[node]._request(node, resource)
return
logger.debug('Appending {} to Request Stack'.format(resource))
request_semaphore = mp.Semaphore(0)
with cls.LOCK:
cls.REQUESTING_STACK.append((resource, request_semaphore))
request_semaphore.acquire()
return
示例8: test_append_kill
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_append_kill(library, mongo_host, library_name):
# Empty DF to start
df = DataFrame({'v': []}, [])
df.index.name = 'index'
library.write('symbol', df)
sem = Semaphore(0)
def run_append(end):
app_1 = Appender(mongo_host, library_name, sem, 0, end)
proc = Process(target=app_1.run)
proc.start()
sem.release()
return proc
def check_written():
sym = library.read('symbol')
print("Checking written %d" % len(sym.data))
# time how long it takes to do an append operation
start = datetime.now()
proc = run_append(1)
proc.join()
check_written()
time_taken = (datetime.now() - start).total_seconds()
for i in range(100):
print("Loop %d" % i)
proc = run_append(100)
# kill it randomly
time.sleep(2 * (random.random() * time_taken))
# Forcibly kill it
proc.terminate()
# Check we can read the data
check_written()
示例9: __init__
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def __init__(
self,
fixers: Fixers,
*args,
interactive: bool = True,
write: bool = False,
silent: bool = False,
in_process: Optional[bool] = None,
hunk_processor: Processor = None,
filename_matcher: Optional[FilenameMatcher] = None,
**kwargs,
) -> None:
options = kwargs.pop("options", {})
super().__init__(fixers, *args, options=options, **kwargs)
self.queue_count = 0
self.queue = multiprocessing.JoinableQueue() # type: ignore
self.results = multiprocessing.Queue() # type: ignore
self.semaphore = multiprocessing.Semaphore(self.NUM_PROCESSES)
self.interactive = interactive
self.write = write
self.silent = silent
if in_process is None:
in_process = self.IN_PROCESS
# pick the most restrictive of flags; we can pickle fixers when
# using spawn.
if sys.platform == "win32" or sys.version_info > (3, 7):
in_process = True
self.in_process = in_process
self.exceptions: List[BowlerException] = []
if hunk_processor is not None:
self.hunk_processor = hunk_processor
else:
self.hunk_processor = lambda f, h: True
self.filename_matcher = filename_matcher or filename_endswith(".py")
示例10: __init__
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def __init__(self, n):
self.n = n
self.counter = SharedCounter(0)
self.barrier = Semaphore(0)
示例11: _parent_of_ignores_sigterm
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def _parent_of_ignores_sigterm(parent_pid, child_pid, setup_done):
def signal_handler(unused_signum, unused_frame):
pass
os.setsid()
signal.signal(signal.SIGTERM, signal_handler)
child_setup_done = multiprocessing.Semaphore(0)
child = multiprocessing.Process(target=TestReapProcessGroup._ignores_sigterm,
args=[child_pid, child_setup_done])
child.start()
child_setup_done.acquire(timeout=5.0)
parent_pid.value = os.getpid()
setup_done.release()
while True:
time.sleep(1)
示例12: test_reap_process_group
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_reap_process_group(self):
"""
Spin up a process that can't be killed by SIGTERM and make sure
it gets killed anyway.
"""
parent_setup_done = multiprocessing.Semaphore(0)
parent_pid = multiprocessing.Value('i', 0)
child_pid = multiprocessing.Value('i', 0)
args = [parent_pid, child_pid, parent_setup_done]
parent = multiprocessing.Process(target=TestReapProcessGroup._parent_of_ignores_sigterm, args=args)
try:
parent.start()
self.assertTrue(parent_setup_done.acquire(timeout=5.0))
self.assertTrue(psutil.pid_exists(parent_pid.value))
self.assertTrue(psutil.pid_exists(child_pid.value))
process_utils.reap_process_group(parent_pid.value, logging.getLogger(), timeout=1)
self.assertFalse(psutil.pid_exists(parent_pid.value))
self.assertFalse(psutil.pid_exists(child_pid.value))
finally:
try:
os.kill(parent_pid.value, signal.SIGKILL) # terminate doesnt work here
os.kill(child_pid.value, signal.SIGKILL) # terminate doesnt work here
except OSError:
pass
示例13: generator
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def generator(self, *args, **kwargs):
""" This function warp generator to ParaWrapper's generator
which is capable of multi-processing
Once the generator function was settled, we can send worker with the task then
work with full-load until meet the buff_size limit
The worker's job is to feed the list and keep it contains more than <buff_size> batches
"""
# Initialization semaphores and numbering
buff_count = Semaphore(value=0)
target_remain = Semaphore(value=self.buff_size)
number = str(self.gen_num)
self.gen_num += 1
# Initializing list
self.batch_list[number] = self.manager.list()
# Assign work and send worker
gen = self.datagen.generator(*args, **kwargs)
worker = Process(target=self.task, args=(gen,number,target_remain, buff_count))
worker.start()
while True:
buff_count.acquire(block=True)
ret = self.batch_list[number].pop()
target_remain.release()
yield ret
示例14: __init__
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def __init__(self, *, process_id, task_id=None,
device_id=None, verbose=100, device_semaphore=None,
uni_process=False):
"""
Handler is provided to functions as a helper object. Currently it could be used to
1. Fancy message printing. The printed message will be labeled with the process id and timestamps,
and will be stored into mailbox, which will be finally returned to the main process, so you don't
need to worry that they will be lost.
2. Device management. Every time you want to use a device, you have to acquire a semaphore from
handler. This is done through calling the handler's use_device_to_run method.
:param int process_id: Process ID.
:param int task_id: Task ID.
:param int device_id: Device ID.
:param int verbose: Verbose level.
:param mp.Semaphore device_semaphore: Device resource semaphore.
:param bool uni_process: Only one process? If so, no need to acquire or release device.
"""
# -1 for main process
self.process_id = process_id
self.task_id = task_id
self.device_id = device_id
# start time
self.verbose = verbose
self.since = time.time()
self.device_semaphore = device_semaphore
self.uni_process = uni_process
# To save the print messages.
self.msg_archive = list()
示例15: __init__
# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def __init__(self, *, identifier, func_to_call, func_args, device_semaphores,
device_usage, device_usage_lock, parent_handler, task_id, verbose,
uni_process=False):
"""
Basically speaking, a task is comprised of a function to call and its args.
A task will provide the function with an additional arg `handler', which could offer some
functionality, including the fancy print function, a device-run function.
:param identifier: Can be any type.
:param func func_to_call: This function will be called when executing the task.
:param dict func_args: The args for func_args.
:param list[mp.Semaphore] device_semaphores: Semaphore for device resource.
:param list device_usage: Recording the usage of each device.
:param mp.Lock device_usage_lock: Lock for device_usage variable.
:param Handler parent_handler: The handler of the main process. This is used to print out some
global information.
:param int task_id: The id for this task.
:param int verbose: Verbose level. 3 is the highest.
:param bool uni_process: If true, there is only one process running now. This is mode is on when
max_processes == 1 in the manager. If this mode is on, the device_run method will not be called.
"""
self.identifier = identifier
self.function_to_call = func_to_call
self.function_args = func_args
self.device_semaphores = device_semaphores
self.device_usage_lock = device_usage_lock
self.device_usage = device_usage
self.n_device = len(device_usage)
self.task_id = task_id
self.parent_handler = parent_handler
self.verbose = verbose
self.uni_process = uni_process