Python multiprocessing.get_start_method函数代码示例

    def transfer_dir_parallel(self, source, destination, jobs):
        """Transfer a directory to the remote side in parallel mode."""
        self.log.debug("Remote transfer: %s -> %s", source, destination)
        self.rmdir(destination)
        if os.path.isdir(source):
            self.mkdir(destination)
            self.log.info("Remote transfer in {} parallel jobs".format(
                jobs))
            self.log.debug("Multiprocessing start method is {}".format(
                multiprocessing.get_start_method()))
            self.log.debug(("Starting multiprocessing Pool for parallel "
                           "remote transfer"))
            with multiprocessing.Pool(jobs) as pool:
                self.log.debug("Starting async for parallel transfer")
                commands = []
                for f in glob.glob1(source, '*'):
                    command = self._copy.split()
                    path = os.path.join(source, f)
                    command.extend([path, '{0}:{1}'.format(
                        self.target_host[0], destination)])
                    commands.append(command)
                results = [
                    pool.apply_async(self._run_command, (cmd,))
                    for cmd in commands
                ]

                self.log.debug("Waiting async results for parallel transfer")
                for r in results:
                    r.get()  # self._run_command returns None
                self.log.debug(("Multiprocessing for parallel transfer "
                               "finished"))
        else:
            raise cdist.Error("Source {} is not a directory".format(source))

    def run(self):
        try:
            self._cmdline.this_module_name = self.__class__.__module__
            if multiprocessing.get_start_method() == 'spawn':
                common.set_global_options(self._cmdline)
                common.sysinit()

            signal.signal(signal.SIGTERM, self._sighandler)
            self.id = self._channel(self._dp_name, self.__class__)
            common.set_current_process(self.id)
            pattern.initialize(self.id)
            if hasattr(self._cmdline, 'clock') and \
               self._cmdline.clock == 'Lamport':
                self._logical_clock = 0
            self._log = logging.getLogger(str(self))
            self._start_comm_thread()
            self._lock = threading.Lock()
            self._lock.acquire()
            self._wait_for_go()

            if not hasattr(self, '_da_run_internal'):
                self._log.error("Process does not have entry point!")
                sys.exit(1)

            result = self._da_run_internal()
            self.report_times()

        except Exception as e:
            sys.stderr.write("Unexpected error at process %s:%r"% (str(self), e))
            traceback.print_tb(e.__traceback__)

        except KeyboardInterrupt as e:
            self._log.debug("Received KeyboardInterrupt, exiting")
            pass

async def qtloop():
    qt_app = None
    qtimport = False
    try:
        import PyQt5.QtCore, PyQt5.QtWidgets
        qtimport = True
    except ImportError:
        pass
    if qtimport:
        if multiprocessing.get_start_method() != "fork":
            print("""Cannot test if Qt can be started
    This is because forking is not possible, you are probably running under Windows
    Starting Qt blindly is not supported, as it may result in segfaults
    """,
            file=sys.stderr)
        else:
            p = Process(target=test_qt)
            p.start()
            p.join()
            if not p.exitcode:
                qt_app = PyQt5.QtWidgets.QApplication(["  "])
    if qt_app is None:
        msg = "Qt could not be started. Qt widgets will not work" #TODO: some kind of env variable to disable this warning
        print(msg,file=sys.stderr)
        return

    while 1:
        qt_app.processEvents()
        await asyncio.sleep(0.01)

def default_test_processes():
    """Default number of test processes when using the --parallel option."""
    # The current implementation of the parallel test runner requires
    # multiprocessing to start subprocesses with fork().
    if multiprocessing.get_start_method() != 'fork':
        return 1
    try:
        return int(os.environ['DJANGO_TEST_PROCESSES'])
    except KeyError:
        return multiprocessing.cpu_count()

    def setUpClass(cls):
        import multiprocessing as mp
        try:
            mp.set_start_method("spawn")
        except RuntimeError:
            pass
        assert mp.get_start_method() == "spawn"

        write_settings()
        cls.app = QApplication([cls.__name__])

def start():
    parser = create_parser()
    args = parser.parse_args()
    config_type = args.type

    config = Config(config_type=config_type)
    setup(config, args)

    logger.info('Config type: %s' % (config_type))
    config.opts.piece_style = args.piece_style
    config.opts.bg_style = args.bg_style
    config.internet.distributed = args.distributed

    if args.cmd == 'self':
        if args.ucci:
            import cchess_alphazero.worker.play_with_ucci_engine as self_play
        else:
            if mp.get_start_method() == 'spawn':
                import cchess_alphazero.worker.self_play_windows as self_play
            else:
                from cchess_alphazero.worker import self_play
        return self_play.start(config)
    elif args.cmd == 'opt':
        from cchess_alphazero.worker import optimize
        return optimize.start(config)
    elif args.cmd == 'play':
        if args.cli:
            import cchess_alphazero.play_games.play_cli as play
        else:
            from cchess_alphazero.play_games import play
        config.opts.light = False
        pwhc = PlayWithHumanConfig()
        pwhc.update_play_config(config.play)
        logger.info(f"AI move first : {args.ai_move_first}")
        play.start(config, not args.ai_move_first)
    elif args.cmd == 'eval':
        if args.elo == False:
            from cchess_alphazero.worker import evaluator
        else:
            import cchess_alphazero.worker.compute_elo as evaluator
        config.eval.update_play_config(config.play)
        evaluator.start(config)
    elif args.cmd == 'sl':
        if args.onegreen:
            import cchess_alphazero.worker.sl_onegreen as sl
            sl.start(config, args.skip)
        else:
            from cchess_alphazero.worker import sl
            sl.start(config)
        
    elif args.cmd == 'ob':
        from cchess_alphazero.play_games import ob_self_play
        pwhc = PlayWithHumanConfig()
        pwhc.update_play_config(config.play)
        ob_self_play.start(config, args.ucci, args.ai_move_first)

 def _run_global_explorers_parallel(self, out_path):
     self.log.debug("Running global explorers in {} parallel jobs".format(
         self.jobs))
     self.log.trace("Multiprocessing start method is {}".format(
         multiprocessing.get_start_method()))
     self.log.trace(("Starting multiprocessing Pool for global "
                    "explorers run"))
     args = [
         (e, out_path, ) for e in self.list_global_explorer_names()
     ]
     mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
     self.log.trace(("Multiprocessing run for global explorers "
                     "finished"))

def initialize_runtime_options(options=None):
    """Sets and sanitizes runtime options.

    'options' should be a dict-like object containing mappings from options
    names to corresponding values.

    """
    import multiprocessing

    from . import compiler

    global GlobalOptions

    if not GlobalOptions:
        GlobalOptions = dict()
    if options:
        GlobalOptions.update(options)

    # Parse '--substitute-classes' and '--substitute-modules':
    GlobalOptions['substitute_classes'] = \
                            _parse_items(GlobalOptions.get('substitute_classes'))
    GlobalOptions['substitute_modules'] = \
                            _parse_items(GlobalOptions.get('substitute_modules'))

    if GlobalOptions.get('nodename') is None:
        GlobalOptions['nodename'] = ''

    _set_hostname()

    # Configure multiprocessing package to use chosen semantics:
    startmeth = GlobalOptions.get('start_method')
    if startmeth != multiprocessing.get_start_method(allow_none=True):
        multiprocessing.set_start_method(startmeth)

    # Convert 'compiler_flags' to a namespace object that can be passed directly
    # to the compiler:
    GlobalOptions['compiler_args'] \
        = compiler.ui.parse_compiler_args(
            GlobalOptions.get('compiler_flags', '').split())

    # Make sure the directory for storing trace files exists:
    if GlobalOptions.get('record_trace'):
        if 'logdir' not in GlobalOptions:
            raise ConfigurationError(
                "'record_trace' enabled without setting 'logdir'")
        os.makedirs(GlobalOptions['logdir'], exist_ok=True)

    def _run_global_explorers_parallel(self, out_path):
        self.log.info("Running global explorers in {} parallel jobs".format(
            self.jobs))
        self.log.debug("Multiprocessing start method is {}".format(
            multiprocessing.get_start_method()))
        self.log.debug(("Starting multiprocessing Pool for global "
                       "explorers run"))
        with multiprocessing.Pool(self.jobs) as pool:
            self.log.debug("Starting async for global explorer run")
            results = [
                pool.apply_async(self._run_global_explorer, (e, out_path,))
                for e in self.list_global_explorer_names()
            ]

            self.log.debug("Waiting async results for global explorer runs")
            for r in results:
                r.get()  # self._run_global_explorer returns None
            self.log.debug(("Multiprocessing run for global explorers "
                           "finished"))

 def _transfer_dir_parallel(self, source, destination, jobs):
     """Transfer a directory to the remote side in parallel mode."""
     self.log.debug("Remote transfer in {} parallel jobs".format(
         jobs))
     self.log.trace("Multiprocessing start method is {}".format(
         multiprocessing.get_start_method()))
     self.log.trace(("Starting multiprocessing Pool for parallel "
                     "remote transfer"))
     args = [
         (command, )
         for command in self._transfer_dir_commands(source, destination)
     ]
     if len(args) == 1:
         self.log.debug("Only one dir entry, transfering sequentially")
         self._run_command(args[0])
     else:
         mp_pool_run(self._run_command, args, jobs=jobs)
     self.log.trace(("Multiprocessing for parallel transfer "
                     "finished"))

    def wrapper(*args, **kwargs):
        future = ProcessFuture()
        reader, writer = Pipe(duplex=False)

        if get_start_method() != 'fork':
            target = _trampoline
            args = [function.__name__, function.__module__] + list(args)
        else:
            target = function

        worker = launch_process(
            _function_handler, target, args, kwargs, writer)

        writer.close()

        future.set_running_or_notify_cancel()

        launch_thread(_worker_handler, future, worker, reader, timeout)

        return future

        # and you should call it again with exc=NULL to revert the effect"""
        ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
        raise SystemError("PyThreadState_SetAsyncExc failed")

class KillableThread(threading.Thread):
    def kill(self, exctype=SystemError):
        if not self.isAlive():
            return
        tid = self.ident
        _async_raise(tid, exctype)
    terminate = kill

USE_PROCESSES = os.environ.get("SEAMLESS_USE_PROCESSES")
if USE_PROCESSES is None:
    USE_PROCESSES = True
    if multiprocessing.get_start_method() != "fork":
        USE_PROCESSES = False
else:
    if USE_PROCESSES == "0" or USE_PROCESSES.upper() == "FALSE":
        USE_PROCESSES = False
if USE_PROCESSES:
    from multiprocessing import JoinableQueue as Queue
    Executor = Process
else:
    from queue import Queue
    Executor = KillableThread

def return_preliminary(result_queue, value):
    #print("return_preliminary", value)
    result_queue.put((-1, value))

def entrypoint():
    GlobalOptions = common.global_options()
    if GlobalOptions.start_method != \
       multiprocessing.get_start_method(allow_none=True):
        multiprocessing.set_start_method(GlobalOptions.start_method)
    target = GlobalOptions.file
    source_dir = os.path.dirname(target)
    basename = strip_suffix(os.path.basename(target))
    if not os.access(target, os.R_OK):
        die("Can not access source file %s" % target)

    sys.path.insert(0, source_dir)
    try:
        module = import_da(basename,
                           from_dir=source_dir,
                           compiler_args=GlobalOptions.compiler_flags.split())
    except ImportError as e:
        die("ImportError: " + str(e))
    if not (hasattr(module, 'main') and
            isinstance(module.main, types.FunctionType)):
        die("'main' function not defined!")
    GlobalOptions.this_module_name = module.__name__
    GlobalOptions.main_module_name = module.__name__
    if GlobalOptions.inc_module_name is None:
        GlobalOptions.inc_module_name = module.__name__ + "_inc"
    common.sysinit()

    # Start the background statistics thread:
    RootLock.acquire()
    stat_th = threading.Thread(target=collect_statistics,
                               name="Stat Thread")
    stat_th.daemon = True
    stat_th.start()

    niters = GlobalOptions.iterations
    stats = {'sent' : 0, 'usrtime': 0, 'systime' : 0, 'time' : 0,
              'units' : 0, 'mem' : 0}
    # Start main program
    sys.argv = [target] + GlobalOptions.args
    try:
        for i in range(0, niters):
            log.info("Running iteration %d ..." % (i+1))

            walltime_start = time.perf_counter()
            module.main()

            print("Waiting for remaining child processes to terminate..."
                  "(Press \"Ctrl-C\" to force kill)")

            walltime = time.perf_counter() - walltime_start

            #log_performance_statistics(walltime)
            r = aggregate_statistics()
            for k, v in r.items():
                stats[k] += v

        for k in stats:
            stats[k] /= niters

        perffd = None
        # if GlobalOptions.perffile is not None:
        #     perffd = open(GlobalOptions.perffile, "w")
        if perffd is not None:
            print_simple_statistics(perffd)
            perffd.close()

        dumpfd = None
        # if GlobalOptions.dumpfile is not None:
        #     dumpfd = open(GlobalOptions.dumpfile, "wb")
        if dumpfd is not None:
            pickle.dump(stats, fd)
            dumpfd.close()

    except KeyboardInterrupt as e:
        log.info("Received keyboard interrupt.")
    except Exception as e:
        err_info = sys.exc_info()
        log.error("Caught unexpected global exception: %r", e)
        traceback.print_tb(err_info[2])

    log.info("Terminating...")

def _real_worker_func(params):
    # In a context where `multiprocessing` is using the `spawn` forking model,
    # the new process doesn't inherit anything, so we lost all our logging
    # configuration here. Let's set it up again.
    if (hasattr(multiprocessing, 'get_start_method') and
            multiprocessing.get_start_method() == 'spawn'):
        from piecrust.main import _pre_parse_chef_args
        _pre_parse_chef_args(sys.argv[1:])

    wid = params.wid
    logger.debug("Worker %d initializing..." % wid)

    # We don't need those.
    params.inqueue._writer.close()
    params.outqueue._reader.close()

    # Initialize the underlying worker class.
    w = params.worker_class(*params.initargs)
    w.wid = wid
    try:
        w.initialize()
    except Exception as ex:
        logger.error("Working failed to initialize:")
        logger.exception(ex)
        params.outqueue.put(None)
        return

    use_threads = False
    if use_threads:
        # Create threads to read/write the jobs and results from/to the
        # main arbitrator process.
        local_job_queue = queue.Queue()
        reader_thread = threading.Thread(
                target=_job_queue_reader,
                args=(params.inqueue.get, local_job_queue),
                name="JobQueueReaderThread")
        reader_thread.start()

        local_result_queue = queue.Queue()
        writer_thread = threading.Thread(
                target=_job_results_writer,
                args=(local_result_queue, params.outqueue.put),
                name="JobResultWriterThread")
        writer_thread.start()

        get = local_job_queue.get
        put = local_result_queue.put_nowait
    else:
        get = params.inqueue.get
        put = params.outqueue.put

    # Start pumping!
    completed = 0
    time_in_get = 0
    time_in_put = 0
    while True:
        get_start_time = time.perf_counter()
        task = get()
        time_in_get += (time.perf_counter() - get_start_time)

        task_type, task_data = task
        if task_type == TASK_END:
            logger.debug("Worker %d got end task, exiting." % wid)
            wprep = {
                    'WorkerTaskGet': time_in_get,
                    'WorkerResultPut': time_in_put}
            try:
                rep = (task_type, True, wid, (wid, w.getReport(wprep)))
            except Exception as e:
                logger.debug("Error getting report: %s" % e)
                if params.wrap_exception:
                    e = multiprocessing.ExceptionWithTraceback(
                            e, e.__traceback__)
                rep = (task_type, False, wid, (wid, e))
            put(rep)
            break

        if task_type == TASK_JOB:
            task_data = (task_data,)

        for t in task_data:
            try:
                res = (TASK_JOB, True, wid, w.process(t))
            except Exception as e:
                if params.wrap_exception:
                    e = multiprocessing.ExceptionWithTraceback(
                            e, e.__traceback__)
                res = (TASK_JOB, False, wid, e)

            put_start_time = time.perf_counter()
            put(res)
            time_in_put += (time.perf_counter() - put_start_time)

            completed += 1

    if use_threads:
        logger.debug("Worker %d waiting for reader/writer threads." % wid)
        local_result_queue.put_nowait(None)
        reader_thread.join()
        writer_thread.join()
#.........这里部分代码省略.........

__author__ = 'fpbatta'

import time

import numpy as np
import sys
import multiprocessing as mp
if __name__ == '__main__':
    #mp.freeze_support()

    print(mp.get_all_start_methods())
    sm = mp.get_start_method()
    print("sm: ", sm)
    mp.set_start_method('forkserver', force=True)
    print("sm 2: ", sm)

    sys.path.append('/home/fpbatta/src/GUI/Plugins')
    sys.path.append('/home/fpbatta/src/GUI/Plugins/multiprocessing_plugin')

    #sys.path.append('/Users/fpbatta/src/GUImerge/GUI/Plugins/multiprocessing_plugin')
    from multiprocessing_plugin.multiprocessing_plugin import MultiprocessingPlugin


    m = MultiprocessingPlugin()

    m.startup(20000.)
    m.bufferfunction(np.random.random((11,1000)))

    for i in range(100):
        m.bufferfunction(200. * np.random.random((11,1000)))
        time.sleep(0.05)