Python multiprocessing.Semaphore方法代码示例

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def initialize_fallback_semaphores():
    """This needs to be called once at the very beginning of starting ACE."""

    # we need some fallback functionality for when the network semaphore server is down
    # these semaphores serve that purpose
    global_engine_instance_count = saq.CONFIG['global'].getint('global_engine_instance_count')
    for key in saq.CONFIG['network_semaphore'].keys():
        if key.startswith('semaphore_'):
            semaphore_name = key[len('semaphore_'):]
            # the configuration settings for the network semaphore specify how many connections
            # are allowed to a specific resource at once, globally
            # so if we unable to coordinate globally, the fall back is to divide the available
            # number of resources between all the engines evenly
            # that's what this next equation is for
            fallback_limit = int(floor(saq.CONFIG['network_semaphore'].getfloat(key) / float(global_engine_instance_count)))
            # we allow a minimum of one per engine
            if fallback_limit < 1:
                fallback_limit = 1

            logging.debug("fallback semaphore count for {0} is {1}".format(semaphore_name, fallback_limit))
            fallback_semaphores[semaphore_name] = LoggingSemaphore(fallback_limit)
            #fallback_semaphores[semaphore_name] = multiprocessing.Semaphore(fallback_limit)
            #fallback_semaphores[semaphore_name].semaphore_name = 'fallback {0}'.format(semaphore_name) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def test_timeout(self):
        if self.TYPE != 'processes':
            self.skipTest('test not appropriate for {}'.format(self.TYPE))

        sem = self.Semaphore(0)
        acquire = TimingWrapper(sem.acquire)

        self.assertEqual(acquire(False), False)
        self.assertTimingAlmostEqual(acquire.elapsed, 0.0)

        self.assertEqual(acquire(False, None), False)
        self.assertTimingAlmostEqual(acquire.elapsed, 0.0)

        self.assertEqual(acquire(False, TIMEOUT1), False)
        self.assertTimingAlmostEqual(acquire.elapsed, 0)

        self.assertEqual(acquire(True, TIMEOUT2), False)
        self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT2)

        self.assertEqual(acquire(timeout=TIMEOUT3), False)
        self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT3) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def test_waitfor_timeout(self):
        # based on test in test/lock_tests.py
        cond = self.Condition()
        state = self.Value('i', 0)
        success = self.Value('i', False)
        sem = self.Semaphore(0)

        p = self.Process(target=self._test_waitfor_timeout_f,
                         args=(cond, state, success, sem))
        p.daemon = True
        p.start()
        self.assertTrue(sem.acquire(timeout=10))

        # Only increment 3 times, so state == 4 is never reached.
        for i in range(3):
            time.sleep(0.01)
            with cond:
                state.value += 1
                cond.notify()

        p.join(5)
        self.assertTrue(success.value) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def build_ctrl(self, world_size):
        """
        Builds several parallel communication mechanisms for controlling the
        workflow across processes.
        """
        opt_throttle = (mp.Barrier(world_size) if world_size > 1 else
            None)
        return AttrDict(
            quit=mp.Value('b', lock=True),
            quit_opt=mp.RawValue('b'),
            sample_ready=[mp.Semaphore(0) for _ in range(2)],  # Double buffer.
            sample_copied=[mp.Semaphore(1) for _ in range(2)],
            sampler_itr=mp.Value('l', lock=True),
            opt_throttle=opt_throttle,
            eval_time=mp.Value('d', lock=True),
        ) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def test_get_size(self):
        """
        Creates a "producer" process that enqueues ``n`` elements
        every ``interval`` seconds. Asserts that a ``get(n, timeout=n*interval+delta)``
        yields all ``n`` elements.
        """
        mp_queue = mp.Queue()
        request_queue = MultiprocessingRequestQueue(mp_queue)
        n = 10
        interval = 0.1
        sem = mp.Semaphore(0)

        p = mp.Process(target=_enqueue_on_interval, args=(mp_queue, n, interval, sem))
        p.start()

        sem.acquire()  # blocks until the process has started to run the function
        timeout = interval * (n + 1)
        start = time.time()
        requests = request_queue.get(n, timeout=timeout)
        self.assertLessEqual(time.time() - start, timeout + interval)
        self.assertEqual(n, len(requests)) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def test_get_less_than_size(self):
        """
        Tests slow producer.
        Creates a "producer" process that enqueues ``n`` elements
        every ``interval`` seconds. Asserts that a ``get(n, timeout=(interval * n/2))``
        yields at most ``n/2`` elements.
        """
        mp_queue = mp.Queue()
        request_queue = MultiprocessingRequestQueue(mp_queue)
        n = 10
        interval = 0.1
        sem = mp.Semaphore(0)

        p = mp.Process(target=_enqueue_on_interval, args=(mp_queue, n, interval, sem))
        p.start()

        sem.acquire()  # blocks until the process has started to run the function
        requests = request_queue.get(n, timeout=(interval * (n / 2)))
        self.assertLessEqual(n / 2, len(requests)) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def _request(cls, resource):
        """ResourceManager, we recommand using scheduler instead of creating your own
        resource manager.
        """
        assert cls.check_possible(resource), \
            'Requested num_cpu={} and num_gpu={} should be less than or equal to' + \
            'largest node availability CPUs={}, GPUs={}'. \
            format(resource.num_cpus, resource.num_gpus, cls.MAX_GPU_COUNT, cls.MAX_CPU_COUNT)
       
        with cls.LOCK:
            node = cls.check_availability(resource)
            if node is not None:
                cls.NODE_RESOURCE_MANAGER[node]._request(node, resource)
                return

        logger.debug('Appending {} to Request Stack'.format(resource))
        request_semaphore = mp.Semaphore(0)
        with cls.LOCK:
            cls.REQUESTING_STACK.append((resource, request_semaphore))
        request_semaphore.acquire()
        return 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def test_append_kill(library, mongo_host, library_name):
    # Empty DF to start
    df = DataFrame({'v': []}, [])
    df.index.name = 'index'
    library.write('symbol', df)

    sem = Semaphore(0)

    def run_append(end):
        app_1 = Appender(mongo_host, library_name, sem, 0, end)
        proc = Process(target=app_1.run)
        proc.start()
        sem.release()
        return proc

    def check_written():
        sym = library.read('symbol')
        print("Checking written %d" % len(sym.data))

    # time how long it takes to do an append operation
    start = datetime.now()
    proc = run_append(1)
    proc.join()
    check_written()
    time_taken = (datetime.now() - start).total_seconds()

    for i in range(100):
        print("Loop %d" % i)
        proc = run_append(100)
        # kill it randomly
        time.sleep(2 * (random.random() * time_taken))
        # Forcibly kill it
        proc.terminate()
        # Check we can read the data
        check_written() 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def __init__(
        self,
        fixers: Fixers,
        *args,
        interactive: bool = True,
        write: bool = False,
        silent: bool = False,
        in_process: Optional[bool] = None,
        hunk_processor: Processor = None,
        filename_matcher: Optional[FilenameMatcher] = None,
        **kwargs,
    ) -> None:
        options = kwargs.pop("options", {})
        super().__init__(fixers, *args, options=options, **kwargs)
        self.queue_count = 0
        self.queue = multiprocessing.JoinableQueue()  # type: ignore
        self.results = multiprocessing.Queue()  # type: ignore
        self.semaphore = multiprocessing.Semaphore(self.NUM_PROCESSES)
        self.interactive = interactive
        self.write = write
        self.silent = silent
        if in_process is None:
            in_process = self.IN_PROCESS
        # pick the most restrictive of flags; we can pickle fixers when
        # using spawn.
        if sys.platform == "win32" or sys.version_info > (3, 7):
            in_process = True
        self.in_process = in_process
        self.exceptions: List[BowlerException] = []
        if hunk_processor is not None:
            self.hunk_processor = hunk_processor
        else:
            self.hunk_processor = lambda f, h: True
        self.filename_matcher = filename_matcher or filename_endswith(".py") 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def __init__(self, n):
        self.n = n
        self.counter = SharedCounter(0)
        self.barrier = Semaphore(0) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def _parent_of_ignores_sigterm(parent_pid, child_pid, setup_done):
        def signal_handler(unused_signum, unused_frame):
            pass
        os.setsid()
        signal.signal(signal.SIGTERM, signal_handler)
        child_setup_done = multiprocessing.Semaphore(0)
        child = multiprocessing.Process(target=TestReapProcessGroup._ignores_sigterm,
                                        args=[child_pid, child_setup_done])
        child.start()
        child_setup_done.acquire(timeout=5.0)
        parent_pid.value = os.getpid()
        setup_done.release()
        while True:
            time.sleep(1) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def test_reap_process_group(self):
        """
        Spin up a process that can't be killed by SIGTERM and make sure
        it gets killed anyway.
        """
        parent_setup_done = multiprocessing.Semaphore(0)
        parent_pid = multiprocessing.Value('i', 0)
        child_pid = multiprocessing.Value('i', 0)
        args = [parent_pid, child_pid, parent_setup_done]
        parent = multiprocessing.Process(target=TestReapProcessGroup._parent_of_ignores_sigterm, args=args)
        try:
            parent.start()
            self.assertTrue(parent_setup_done.acquire(timeout=5.0))
            self.assertTrue(psutil.pid_exists(parent_pid.value))
            self.assertTrue(psutil.pid_exists(child_pid.value))

            process_utils.reap_process_group(parent_pid.value, logging.getLogger(), timeout=1)

            self.assertFalse(psutil.pid_exists(parent_pid.value))
            self.assertFalse(psutil.pid_exists(child_pid.value))
        finally:
            try:
                os.kill(parent_pid.value, signal.SIGKILL)  # terminate doesnt work here
                os.kill(child_pid.value, signal.SIGKILL)  # terminate doesnt work here
            except OSError:
                pass 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def generator(self, *args, **kwargs):
        """ This function warp generator to ParaWrapper's generator
            which is capable of multi-processing
            Once the generator function was settled, we can send worker with the task then
            work with full-load until meet the buff_size limit

            The worker's job is to feed the list and keep it contains more than <buff_size> batches
        """
        #   Initialization semaphores and numbering
        buff_count = Semaphore(value=0)
        target_remain = Semaphore(value=self.buff_size)
        number = str(self.gen_num)
        self.gen_num += 1

        #   Initializing list
        self.batch_list[number] = self.manager.list()

        #   Assign work and send worker
        gen = self.datagen.generator(*args, **kwargs)
        worker = Process(target=self.task, args=(gen,number,target_remain, buff_count))
        worker.start()

        while True:
            buff_count.acquire(block=True)
            ret = self.batch_list[number].pop()
            target_remain.release()
            yield ret 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def __init__(self, *, process_id, task_id=None,
                 device_id=None, verbose=100, device_semaphore=None,
                 uni_process=False):
        """
        Handler is provided to functions as a helper object. Currently it could be used to
        1. Fancy message printing. The printed message will be labeled with the process id and timestamps,
        and will be stored into mailbox, which will be finally returned to the main process, so you don't
        need to worry that they will be lost.
        2. Device management. Every time you want to use a device, you have to acquire a semaphore from
        handler. This is done through calling the handler's use_device_to_run method.
        :param int process_id: Process ID.
        :param int task_id: Task ID.
        :param int device_id: Device ID.
        :param int verbose: Verbose level.
        :param mp.Semaphore device_semaphore: Device resource semaphore.
        :param bool uni_process: Only one process? If so, no need to acquire or release device.
        """

        # -1 for main process
        self.process_id = process_id
        self.task_id = task_id
        self.device_id = device_id
        # start time
        self.verbose = verbose
        self.since = time.time()
        self.device_semaphore = device_semaphore
        self.uni_process = uni_process

        # To save the print messages.
        self.msg_archive = list() 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Semaphore [as 别名]
def __init__(self, *, identifier, func_to_call, func_args, device_semaphores,
                 device_usage, device_usage_lock, parent_handler, task_id, verbose,
                 uni_process=False):
        """
        Basically speaking, a task is comprised of a function to call and its args.
        A task will provide the function with an additional arg `handler', which could offer some
        functionality, including the fancy print function, a device-run function.
        :param identifier: Can be any type.
        :param func func_to_call: This function will be called when executing the task.
        :param dict func_args: The args for func_args.
        :param list[mp.Semaphore] device_semaphores: Semaphore for device resource.
        :param list device_usage: Recording the usage of each device.
        :param mp.Lock device_usage_lock: Lock for device_usage variable.
        :param Handler parent_handler: The handler of the main process. This is used to print out some
        global information.
        :param int task_id: The id for this task.
        :param int verbose: Verbose level. 3 is the highest.
        :param bool uni_process: If true, there is only one process running now. This is mode is on when
        max_processes == 1 in the manager. If this mode is on, the device_run method will not be called.
        """
        self.identifier = identifier
        self.function_to_call = func_to_call
        self.function_args = func_args

        self.device_semaphores = device_semaphores
        self.device_usage_lock = device_usage_lock
        self.device_usage = device_usage
        self.n_device = len(device_usage)

        self.task_id = task_id
        self.parent_handler = parent_handler
        self.verbose = verbose

        self.uni_process = uni_process