Python multiprocessing.queues方法代码示例

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import queues [as 别名]
def add_evaluator_workers(self):
        """Add evaluator workers

        Evaluator workers receive all graph updates, hence are updated.
        Each evaluator worker holds an enabled scenario-evaluator and process
        every change.
        Each worker's scenario-evaluator runs different template scenarios.
        Interface to these workers is:
        submit_graph_update(..)
        submit_start_evaluations(..)
        submit_evaluators_reload_templates(..)
        """
        if self._evaluator_queues:
            raise VitrageError('add_evaluator_workers called more than once')
        workers = CONF.evaluator.workers
        queues = [multiprocessing.JoinableQueue() for i in range(workers)]
        self.add(EvaluatorWorker,
                 args=(queues, workers),
                 workers=workers)
        self._evaluator_queues = queues
        self._all_queues.extend(queues) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import queues [as 别名]
def lenient(self, lenient):
        if lenient not in (False, True):
            raise ValueError("Invalid lenient value: {}".format(lenient))
        self.__lenient = lenient

    # @property
    # def submission_queue(self):
    #     return self.__submission_queue
    #
    # def __set_submission_queue(self, submission):
    #     if isinstance(submission, multiprocessing.queues.SimpleQueue):
    #         if sys.version_info.minor < 6:
    #             raise TypeError("Invalid queue object for Python 3.5 and earlier!")
    #         submission.put_nowait = submission.put
    #     elif not isinstance(submission, (multiprocessing.queues.Queue,
    #                                    queue.Queue)):
    #         raise TypeError("Invalid queue object: {}".format(type(submission)))
    #     self.__submission_queue = submission
    # 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import queues [as 别名]
def add_api_workers(self):
        """Add Api workers

        Api workers receive all graph updates, hence are updated.
        Each template worker holds a disabled scenario-evaluator that does
        not process changes.
        These also hold a rpc server and process the incoming Api calls
        """
        if self._api_queues:
            raise VitrageError('add_api_workers called more than once')
        workers = CONF.api.workers
        queues = [multiprocessing.Queue() for i in range(workers)]
        self.add(ApiWorker, args=(queues,), workers=workers)
        self._api_queues = queues
        self._all_queues.extend(queues) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import queues [as 别名]
def _submit_and_wait(queues, payload):
        for q in queues:
            q.put(payload)
        for q in queues:
            if isinstance(q, multiprocessing.queues.JoinableQueue):
                q.join() 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import queues [as 别名]
def _read_queue(self):
        LOG.debug("%s - reading queue %s",
                  self.__class__.__name__, self.worker_id)
        while True:
            try:
                next_task = self._task_queue.get()
                self.do_task(next_task)
            except Exception:
                LOG.exception("Graph may not be in sync.")
            if isinstance(self._task_queue,
                          multiprocessing.queues.JoinableQueue):
                self._task_queue.task_done() 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import queues [as 别名]
def pool_worker_main(item: WorkItemInput, output: multiprocessing.queues.Queue) -> None:
    try:
        # TODO figure out a more reliable way to suppress this. Redirect output?
        # Ignore ctrl-c in workers to reduce noisy tracebacks (the parent will kill us):
        signal.signal(signal.SIGINT, signal.SIG_IGN)

        if hasattr(os, 'nice'): # analysis should run at a low priority
            os.nice(10)
        set_debug(False)
        filename, options, deadline = item
        stats: Counter[str] = Counter()
        options.stats = stats
        _, module_name = extract_module_from_file(filename)
        try:
            module = load_by_qualname(module_name)
        except NotFound:
            return
        except ErrorDuringImport as e:
            output.put((filename, stats, [import_error_msg(e)]))
            debug(f'Not analyzing "{filename}" because import failed: {e}')
            return
        messages = analyze_any(module, options)
        output.put((filename, stats, messages))
    except BaseException as e:
        raise CrosshairInternal(
            'Worker failed while analyzing ' + filename) from e 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import queues [as 别名]
def __set_logging_queue(self, logging_queue):
        if isinstance(logging_queue, multiprocessing.queues.SimpleQueue):
            if sys.version_info.minor < 6:
                raise TypeError("Invalid queue object for Python 3.5 and earlier!")
            logging_queue.put_nowait = logging_queue.put
        elif not isinstance(logging_queue, (multiprocessing.queues.Queue, queue.Queue)):
            raise TypeError("Invalid queue object: {}".format(type(logging_queue)))
        self.__logging_queue = logging_queue