當前位置: 首頁>>代碼示例>>Python>>正文


Python multiprocessing.Semaphore方法代碼示例

本文整理匯總了Python中multiprocessing.Semaphore方法的典型用法代碼示例。如果您正苦於以下問題:Python multiprocessing.Semaphore方法的具體用法?Python multiprocessing.Semaphore怎麽用?Python multiprocessing.Semaphore使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在multiprocessing的用法示例。


在下文中一共展示了multiprocessing.Semaphore方法的15個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Python代碼示例。

示例1: initialize_fallback_semaphores

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def initialize_fallback_semaphores():
    """This needs to be called once at the very beginning of starting ACE."""

    # we need some fallback functionality for when the network semaphore server is down
    # these semaphores serve that purpose
    global_engine_instance_count = saq.CONFIG['global'].getint('global_engine_instance_count')
    for key in saq.CONFIG['network_semaphore'].keys():
        if key.startswith('semaphore_'):
            semaphore_name = key[len('semaphore_'):]
            # the configuration settings for the network semaphore specify how many connections
            # are allowed to a specific resource at once, globally
            # so if we unable to coordinate globally, the fall back is to divide the available
            # number of resources between all the engines evenly
            # that's what this next equation is for
            fallback_limit = int(floor(saq.CONFIG['network_semaphore'].getfloat(key) / float(global_engine_instance_count)))
            # we allow a minimum of one per engine
            if fallback_limit < 1:
                fallback_limit = 1

            logging.debug("fallback semaphore count for {0} is {1}".format(semaphore_name, fallback_limit))
            fallback_semaphores[semaphore_name] = LoggingSemaphore(fallback_limit)
            #fallback_semaphores[semaphore_name] = multiprocessing.Semaphore(fallback_limit)
            #fallback_semaphores[semaphore_name].semaphore_name = 'fallback {0}'.format(semaphore_name) 
開發者ID:IntegralDefense,項目名稱:ACE,代碼行數:25,代碼來源:network_semaphore.py

示例2: test_timeout

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_timeout(self):
        if self.TYPE != 'processes':
            self.skipTest('test not appropriate for {}'.format(self.TYPE))

        sem = self.Semaphore(0)
        acquire = TimingWrapper(sem.acquire)

        self.assertEqual(acquire(False), False)
        self.assertTimingAlmostEqual(acquire.elapsed, 0.0)

        self.assertEqual(acquire(False, None), False)
        self.assertTimingAlmostEqual(acquire.elapsed, 0.0)

        self.assertEqual(acquire(False, TIMEOUT1), False)
        self.assertTimingAlmostEqual(acquire.elapsed, 0)

        self.assertEqual(acquire(True, TIMEOUT2), False)
        self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT2)

        self.assertEqual(acquire(timeout=TIMEOUT3), False)
        self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT3) 
開發者ID:Microvellum,項目名稱:Fluid-Designer,代碼行數:23,代碼來源:_test_multiprocessing.py

示例3: test_waitfor_timeout

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_waitfor_timeout(self):
        # based on test in test/lock_tests.py
        cond = self.Condition()
        state = self.Value('i', 0)
        success = self.Value('i', False)
        sem = self.Semaphore(0)

        p = self.Process(target=self._test_waitfor_timeout_f,
                         args=(cond, state, success, sem))
        p.daemon = True
        p.start()
        self.assertTrue(sem.acquire(timeout=10))

        # Only increment 3 times, so state == 4 is never reached.
        for i in range(3):
            time.sleep(0.01)
            with cond:
                state.value += 1
                cond.notify()

        p.join(5)
        self.assertTrue(success.value) 
開發者ID:Microvellum,項目名稱:Fluid-Designer,代碼行數:24,代碼來源:_test_multiprocessing.py

示例4: build_ctrl

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def build_ctrl(self, world_size):
        """
        Builds several parallel communication mechanisms for controlling the
        workflow across processes.
        """
        opt_throttle = (mp.Barrier(world_size) if world_size > 1 else
            None)
        return AttrDict(
            quit=mp.Value('b', lock=True),
            quit_opt=mp.RawValue('b'),
            sample_ready=[mp.Semaphore(0) for _ in range(2)],  # Double buffer.
            sample_copied=[mp.Semaphore(1) for _ in range(2)],
            sampler_itr=mp.Value('l', lock=True),
            opt_throttle=opt_throttle,
            eval_time=mp.Value('d', lock=True),
        ) 
開發者ID:astooke,項目名稱:rlpyt,代碼行數:18,代碼來源:async_rl.py

示例5: test_get_size

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_get_size(self):
        """
        Creates a "producer" process that enqueues ``n`` elements
        every ``interval`` seconds. Asserts that a ``get(n, timeout=n*interval+delta)``
        yields all ``n`` elements.
        """
        mp_queue = mp.Queue()
        request_queue = MultiprocessingRequestQueue(mp_queue)
        n = 10
        interval = 0.1
        sem = mp.Semaphore(0)

        p = mp.Process(target=_enqueue_on_interval, args=(mp_queue, n, interval, sem))
        p.start()

        sem.acquire()  # blocks until the process has started to run the function
        timeout = interval * (n + 1)
        start = time.time()
        requests = request_queue.get(n, timeout=timeout)
        self.assertLessEqual(time.time() - start, timeout + interval)
        self.assertEqual(n, len(requests)) 
開發者ID:pytorch,項目名稱:elastic,代碼行數:23,代碼來源:local_timer_test.py

示例6: test_get_less_than_size

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_get_less_than_size(self):
        """
        Tests slow producer.
        Creates a "producer" process that enqueues ``n`` elements
        every ``interval`` seconds. Asserts that a ``get(n, timeout=(interval * n/2))``
        yields at most ``n/2`` elements.
        """
        mp_queue = mp.Queue()
        request_queue = MultiprocessingRequestQueue(mp_queue)
        n = 10
        interval = 0.1
        sem = mp.Semaphore(0)

        p = mp.Process(target=_enqueue_on_interval, args=(mp_queue, n, interval, sem))
        p.start()

        sem.acquire()  # blocks until the process has started to run the function
        requests = request_queue.get(n, timeout=(interval * (n / 2)))
        self.assertLessEqual(n / 2, len(requests)) 
開發者ID:pytorch,項目名稱:elastic,代碼行數:21,代碼來源:local_timer_test.py

示例7: _request

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def _request(cls, resource):
        """ResourceManager, we recommand using scheduler instead of creating your own
        resource manager.
        """
        assert cls.check_possible(resource), \
            'Requested num_cpu={} and num_gpu={} should be less than or equal to' + \
            'largest node availability CPUs={}, GPUs={}'. \
            format(resource.num_cpus, resource.num_gpus, cls.MAX_GPU_COUNT, cls.MAX_CPU_COUNT)
       
        with cls.LOCK:
            node = cls.check_availability(resource)
            if node is not None:
                cls.NODE_RESOURCE_MANAGER[node]._request(node, resource)
                return

        logger.debug('Appending {} to Request Stack'.format(resource))
        request_semaphore = mp.Semaphore(0)
        with cls.LOCK:
            cls.REQUESTING_STACK.append((resource, request_semaphore))
        request_semaphore.acquire()
        return 
開發者ID:awslabs,項目名稱:autogluon,代碼行數:23,代碼來源:dist_manager.py

示例8: test_append_kill

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_append_kill(library, mongo_host, library_name):
    # Empty DF to start
    df = DataFrame({'v': []}, [])
    df.index.name = 'index'
    library.write('symbol', df)

    sem = Semaphore(0)

    def run_append(end):
        app_1 = Appender(mongo_host, library_name, sem, 0, end)
        proc = Process(target=app_1.run)
        proc.start()
        sem.release()
        return proc

    def check_written():
        sym = library.read('symbol')
        print("Checking written %d" % len(sym.data))

    # time how long it takes to do an append operation
    start = datetime.now()
    proc = run_append(1)
    proc.join()
    check_written()
    time_taken = (datetime.now() - start).total_seconds()

    for i in range(100):
        print("Loop %d" % i)
        proc = run_append(100)
        # kill it randomly
        time.sleep(2 * (random.random() * time_taken))
        # Forcibly kill it
        proc.terminate()
        # Check we can read the data
        check_written() 
開發者ID:man-group,項目名稱:arctic,代碼行數:37,代碼來源:test_concurrent_append.py

示例9: __init__

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def __init__(
        self,
        fixers: Fixers,
        *args,
        interactive: bool = True,
        write: bool = False,
        silent: bool = False,
        in_process: Optional[bool] = None,
        hunk_processor: Processor = None,
        filename_matcher: Optional[FilenameMatcher] = None,
        **kwargs,
    ) -> None:
        options = kwargs.pop("options", {})
        super().__init__(fixers, *args, options=options, **kwargs)
        self.queue_count = 0
        self.queue = multiprocessing.JoinableQueue()  # type: ignore
        self.results = multiprocessing.Queue()  # type: ignore
        self.semaphore = multiprocessing.Semaphore(self.NUM_PROCESSES)
        self.interactive = interactive
        self.write = write
        self.silent = silent
        if in_process is None:
            in_process = self.IN_PROCESS
        # pick the most restrictive of flags; we can pickle fixers when
        # using spawn.
        if sys.platform == "win32" or sys.version_info > (3, 7):
            in_process = True
        self.in_process = in_process
        self.exceptions: List[BowlerException] = []
        if hunk_processor is not None:
            self.hunk_processor = hunk_processor
        else:
            self.hunk_processor = lambda f, h: True
        self.filename_matcher = filename_matcher or filename_endswith(".py") 
開發者ID:facebookincubator,項目名稱:Bowler,代碼行數:36,代碼來源:tool.py

示例10: __init__

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def __init__(self, n):
        self.n = n
        self.counter = SharedCounter(0)
        self.barrier = Semaphore(0) 
開發者ID:traai,項目名稱:async-deep-rl,代碼行數:6,代碼來源:shared_utils.py

示例11: _parent_of_ignores_sigterm

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def _parent_of_ignores_sigterm(parent_pid, child_pid, setup_done):
        def signal_handler(unused_signum, unused_frame):
            pass
        os.setsid()
        signal.signal(signal.SIGTERM, signal_handler)
        child_setup_done = multiprocessing.Semaphore(0)
        child = multiprocessing.Process(target=TestReapProcessGroup._ignores_sigterm,
                                        args=[child_pid, child_setup_done])
        child.start()
        child_setup_done.acquire(timeout=5.0)
        parent_pid.value = os.getpid()
        setup_done.release()
        while True:
            time.sleep(1) 
開發者ID:apache,項目名稱:airflow,代碼行數:16,代碼來源:test_process_utils.py

示例12: test_reap_process_group

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def test_reap_process_group(self):
        """
        Spin up a process that can't be killed by SIGTERM and make sure
        it gets killed anyway.
        """
        parent_setup_done = multiprocessing.Semaphore(0)
        parent_pid = multiprocessing.Value('i', 0)
        child_pid = multiprocessing.Value('i', 0)
        args = [parent_pid, child_pid, parent_setup_done]
        parent = multiprocessing.Process(target=TestReapProcessGroup._parent_of_ignores_sigterm, args=args)
        try:
            parent.start()
            self.assertTrue(parent_setup_done.acquire(timeout=5.0))
            self.assertTrue(psutil.pid_exists(parent_pid.value))
            self.assertTrue(psutil.pid_exists(child_pid.value))

            process_utils.reap_process_group(parent_pid.value, logging.getLogger(), timeout=1)

            self.assertFalse(psutil.pid_exists(parent_pid.value))
            self.assertFalse(psutil.pid_exists(child_pid.value))
        finally:
            try:
                os.kill(parent_pid.value, signal.SIGKILL)  # terminate doesnt work here
                os.kill(child_pid.value, signal.SIGKILL)  # terminate doesnt work here
            except OSError:
                pass 
開發者ID:apache,項目名稱:airflow,代碼行數:28,代碼來源:test_process_utils.py

示例13: generator

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def generator(self, *args, **kwargs):
        """ This function warp generator to ParaWrapper's generator
            which is capable of multi-processing
            Once the generator function was settled, we can send worker with the task then
            work with full-load until meet the buff_size limit

            The worker's job is to feed the list and keep it contains more than <buff_size> batches
        """
        #   Initialization semaphores and numbering
        buff_count = Semaphore(value=0)
        target_remain = Semaphore(value=self.buff_size)
        number = str(self.gen_num)
        self.gen_num += 1

        #   Initializing list
        self.batch_list[number] = self.manager.list()

        #   Assign work and send worker
        gen = self.datagen.generator(*args, **kwargs)
        worker = Process(target=self.task, args=(gen,number,target_remain, buff_count))
        worker.start()

        while True:
            buff_count.acquire(block=True)
            ret = self.batch_list[number].pop()
            target_remain.release()
            yield ret 
開發者ID:mpskex,項目名稱:Convolutional-Pose-Machine-tf,代碼行數:29,代碼來源:datagen_para_branch.py

示例14: __init__

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def __init__(self, *, process_id, task_id=None,
                 device_id=None, verbose=100, device_semaphore=None,
                 uni_process=False):
        """
        Handler is provided to functions as a helper object. Currently it could be used to
        1. Fancy message printing. The printed message will be labeled with the process id and timestamps,
        and will be stored into mailbox, which will be finally returned to the main process, so you don't
        need to worry that they will be lost.
        2. Device management. Every time you want to use a device, you have to acquire a semaphore from
        handler. This is done through calling the handler's use_device_to_run method.
        :param int process_id: Process ID.
        :param int task_id: Task ID.
        :param int device_id: Device ID.
        :param int verbose: Verbose level.
        :param mp.Semaphore device_semaphore: Device resource semaphore.
        :param bool uni_process: Only one process? If so, no need to acquire or release device.
        """

        # -1 for main process
        self.process_id = process_id
        self.task_id = task_id
        self.device_id = device_id
        # start time
        self.verbose = verbose
        self.since = time.time()
        self.device_semaphore = device_semaphore
        self.uni_process = uni_process

        # To save the print messages.
        self.msg_archive = list() 
開發者ID:HMEIatJHU,項目名稱:neural-hawkes-particle-smoothing,代碼行數:32,代碼來源:handler.py

示例15: __init__

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import Semaphore [as 別名]
def __init__(self, *, identifier, func_to_call, func_args, device_semaphores,
                 device_usage, device_usage_lock, parent_handler, task_id, verbose,
                 uni_process=False):
        """
        Basically speaking, a task is comprised of a function to call and its args.
        A task will provide the function with an additional arg `handler', which could offer some
        functionality, including the fancy print function, a device-run function.
        :param identifier: Can be any type.
        :param func func_to_call: This function will be called when executing the task.
        :param dict func_args: The args for func_args.
        :param list[mp.Semaphore] device_semaphores: Semaphore for device resource.
        :param list device_usage: Recording the usage of each device.
        :param mp.Lock device_usage_lock: Lock for device_usage variable.
        :param Handler parent_handler: The handler of the main process. This is used to print out some
        global information.
        :param int task_id: The id for this task.
        :param int verbose: Verbose level. 3 is the highest.
        :param bool uni_process: If true, there is only one process running now. This is mode is on when
        max_processes == 1 in the manager. If this mode is on, the device_run method will not be called.
        """
        self.identifier = identifier
        self.function_to_call = func_to_call
        self.function_args = func_args

        self.device_semaphores = device_semaphores
        self.device_usage_lock = device_usage_lock
        self.device_usage = device_usage
        self.n_device = len(device_usage)

        self.task_id = task_id
        self.parent_handler = parent_handler
        self.verbose = verbose

        self.uni_process = uni_process 
開發者ID:HMEIatJHU,項目名稱:neural-hawkes-particle-smoothing,代碼行數:36,代碼來源:task.py


注:本文中的multiprocessing.Semaphore方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台,相關代碼片段篩選自各路編程大神貢獻的開源項目,源碼版權歸原作者所有,傳播和使用請參考對應項目的License;未經允許,請勿轉載。