Python process.BrokenProcessPool方法代码示例

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import BrokenProcessPool [as 别名]
def post(self, message):
        with self._shutdown_lock:
            if self._broken:
                raise process.BrokenProcessPool(
                    'A child process terminated '
                    'abruptly, the process pool is not usable anymore')
            if self._shutdown_thread:
                raise RuntimeError('cannot schedule new futures after shutdown')

            f = _base.Future()
            w = _WorkItem(f, message)

            self._pending_work_items[self._queue_count] = w
            self._work_ids.put(self._queue_count)
            self._queue_count += 1
            # Wake up queue management thread
            self._result_queue.put(None)

            self._start_queue_management_thread()
            return f 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import BrokenProcessPool [as 别名]
def process(self, item: ITEM) -> bool:
        """Applies the filters to an item"""
        try:
            for filt in self.filters:
                await filt.apply(item)
        except asyncio.CancelledError:
            raise
        except EndProcessingItem as item_error:
            item_error.log(logger)
            raise
        except BrokenExecutor:
            logger.exception("Fatal exception while processing %s", item)
            # can't fix this - if one of the pools is done for, so are we
            raise
        except Exception:  # pylint: disable=broad-except
            if not self._shutting_down:
                logger.exception("While processing %s", item)
            return False
        return True 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import BrokenProcessPool [as 别名]
def test_compat_with_concurrent_futures_exception(self):
        # It should be possible to use a loky process pool executor as a dropin
        # replacement for a ProcessPoolExecutor, including when catching
        # exceptions:
        concurrent = pytest.importorskip('concurrent')
        from concurrent.futures.process import BrokenProcessPool as BPPExc

        with pytest.raises(BPPExc):
            get_reusable_executor(max_workers=2).submit(crash).result()
        e = get_reusable_executor(max_workers=2)
        f = e.submit(id, 42)

        # Ensure that loky.Future are compatible with concurrent.futures
        # (see #155)
        assert isinstance(f, concurrent.futures.Future)
        (done, running) = concurrent.futures.wait([f], timeout=15)
        assert len(running) == 0 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import BrokenProcessPool [as 别名]
def handling_broken_process_pool():
    """Handle BrokenProcessPool error."""
    if sys.version_info < (3, 3):
        yield
    else:
        from concurrent.futures.process import BrokenProcessPool
        try:
            yield
        except BrokenProcessPool:
            raise KeyboardInterrupt() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import BrokenProcessPool [as 别名]
def test_killed_child(self):
        # When a child process is abruptly terminated, the whole pool gets
        # "broken".
        futures = [self.executor.submit(time.sleep, 3)]
        # Get one of the processes, and terminate (kill) it
        p = next(iter(self.executor._processes.values()))
        p.terminate()
        for fut in futures:
            self.assertRaises(BrokenProcessPool, fut.result)
        # Submitting other jobs fails as well.
        self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import BrokenProcessPool [as 别名]
def test_shutdown_deadlock(self):
        # Test that the pool calling shutdown do not cause deadlock
        # if a worker fails after the shutdown call.
        self.executor.shutdown(wait=True)
        with self.executor_type(max_workers=2,
                                mp_context=get_context(self.ctx)) as executor:
            self.executor = executor  # Allow clean up in fail_on_deadlock
            f = executor.submit(_crash, delay=.1)
            executor.shutdown(wait=True)
            with self.assertRaises(BrokenProcessPool):
                f.result() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import BrokenProcessPool [as 别名]
def run(self):
        """
        Start the worker

        `run()` will trap signals, on the first ctrl-c it will try to gracefully shut the worker down, waiting up to 15
        seconds for in progress tasks to complete.

        If after 30 seconds tasks are still running, they are forced to terminate and the worker will close.

        This method is blocking -- it will only return when the worker has shutdown, either by control-c or by
        terminating it from the Faktory Web UI.

        :return:
        :rtype:
        """
        # create a pool of workers
        if not self.faktory.is_connected:
            self.faktory.connect(worker_id=self.worker_id)

        self.log.debug(
            "Creating a worker pool with concurrency of {}".format(self.concurrency)
        )

        self._last_heartbeat = datetime.now() + timedelta(
            seconds=self.send_heartbeat_every
        )  # schedule a heartbeat for the future

        self.log.info("Queues: {}".format(", ".join(self.get_queues())))
        self.log.info("Labels: {}".format(", ".join(self.faktory.labels)))

        while True:
            try:
                # tick runs continuously to process events from the faktory connection
                self.tick()
                if not self.faktory.is_connected:
                    break
            except KeyboardInterrupt as e:
                # 1st time through: soft close, wait 15 seconds for jobs to finish and send the work results to faktory
                # 2nd time through: force close, don't wait, fail all current jobs and quit as quickly as possible
                if self.is_disconnecting:
                    break

                self.log.info(
                    "Shutdown: waiting up to 15 seconds for workers to finish current tasks"
                )
                self.disconnect(wait=15)
            except (BrokenProcessPool, BrokenThreadPool):
                self.log.info("Shutting down due to pool failure")
                self.disconnect(force=True, wait=15)
                break

        if self.faktory.is_connected:
            self.log.warning("Forcing worker processes to shutdown...")
            self.disconnect(force=True)

        self.executor.shutdown(wait=False)
        sys.exit(1) 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import BrokenProcessPool [as 别名]
def test_crash(self):
        # extensive testing for deadlock caused by crashes in a pool.
        self.executor.shutdown(wait=True)
        crash_cases = [
            # Check problem occurring while pickling a task in
            # the task_handler thread
            (id, (ErrorAtPickle(),), PicklingError, "error at task pickle"),
            # Check problem occurring while unpickling a task on workers
            (id, (ExitAtUnpickle(),), BrokenProcessPool,
             "exit at task unpickle"),
            (id, (ErrorAtUnpickle(),), BrokenProcessPool,
             "error at task unpickle"),
            (id, (CrashAtUnpickle(),), BrokenProcessPool,
             "crash at task unpickle"),
            # Check problem occurring during func execution on workers
            (_crash, (), BrokenProcessPool,
             "crash during func execution on worker"),
            (_exit, (), SystemExit,
             "exit during func execution on worker"),
            (_raise_error, (RuntimeError, ), RuntimeError,
             "error during func execution on worker"),
            # Check problem occurring while pickling a task result
            # on workers
            (_return_instance, (CrashAtPickle,), BrokenProcessPool,
             "crash during result pickle on worker"),
            (_return_instance, (ExitAtPickle,), SystemExit,
             "exit during result pickle on worker"),
            (_return_instance, (ErrorAtPickle,), PicklingError,
             "error during result pickle on worker"),
            # Check problem occurring while unpickling a task in
            # the result_handler thread
            (_return_instance, (ErrorAtUnpickle,), BrokenProcessPool,
             "error during result unpickle in result_handler"),
            (_return_instance, (ExitAtUnpickle,), BrokenProcessPool,
             "exit during result unpickle in result_handler")
        ]
        for func, args, error, name in crash_cases:
            with self.subTest(name):
                # The captured_stderr reduces the noise in the test report
                with test.support.captured_stderr():
                    executor = self.executor_type(
                        max_workers=2, mp_context=get_context(self.ctx))
                    res = executor.submit(func, *args)
                    with self.assertRaises(error):
                        try:
                            res.result(timeout=self.TIMEOUT)
                        except futures.TimeoutError:
                            # If we did not recover before TIMEOUT seconds,
                            # consider that the executor is in a deadlock state
                            self._fail_on_deadlock(executor)
                    executor.shutdown(wait=True)