Python multiprocessing.get_start_method方法代碼示例

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def start_proc_mask_signal(proc):
    """
    Start process(es) with SIGINT ignored.

    Args:
        proc: (mp.Process or list)

    Note:
        The signal mask is only applied when called from main thread.
    """
    if not isinstance(proc, list):
        proc = [proc]

    with mask_sigint():
        for p in proc:
            if isinstance(p, mp.Process):
                if sys.version_info < (3, 4) or mp.get_start_method() == 'fork':
                    log_once("""
Starting a process with 'fork' method is efficient but not safe and may cause deadlock or crash.
Use 'forkserver' or 'spawn' method instead if you run into such issues.
See https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods on how to set them.
""".replace("\n", ""),
'warn')  # noqa
            p.start() 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def main(args):
    with open(args.config) as f:
        if version.parse(yaml.version >= "5.1"):
            config = yaml.load(f, Loader=yaml.FullLoader)
        else:
            config = yaml.load(f)

    for k, v in config.items():
        setattr(args, k, v)

    # exp path
    if not hasattr(args, 'exp_path'):
        args.exp_path = os.path.dirname(args.config)

    # dist init
    if mp.get_start_method(allow_none=True) != 'spawn':
        mp.set_start_method('spawn', force=True)
    dist_init(args.launcher, backend='nccl')

    # train
    trainer = Trainer(args)
    trainer.run() 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def start_proc_mask_signal(proc):
    """
    Start process(es) with SIGINT ignored.

    Args:
        proc: (mp.Process or list)

    Note:
        The signal mask is only applied when called from main thread.
    """
    if not isinstance(proc, list):
        proc = [proc]

    with mask_sigint():
        for p in proc:
            if isinstance(p, mp.Process):
                if sys.version_info < (3, 4) or mp.get_start_method() == 'fork':
                    log_once(
"Starting a process with 'fork' method is not safe and may consume unnecessary extra CPU memory."
" Use 'forkserver' or 'spawn' method (available after Py3.4) instead if you run into any issues. "
"See https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods on how to set them.",
'warn')  # noqa
            p.start() 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def test_interactive_contex_no_main(self):
        # Ensure that loky context is working properly
        code = '\n'.join([
            'from loky.backend import get_context',
            'ctx = get_context()',
            'assert ctx.get_start_method() == "loky"',
            'p = ctx.Process(target=id, args=(1,))',
            'p.start()',
            'p.join()',
            'msg = "loky context failed to load without safeguard"',
            'assert p.exitcode == 0, msg',
            'print("ok")'
        ])
        try:
            fid, filename = mkstemp(suffix="_joblib.py")
            os.close(fid)
            with open(filename, mode='wb') as f:
                f.write(code.encode('ascii'))
            check_subprocess_call([sys.executable, filename],
                                  stdout_regex=r'ok', timeout=10)
        finally:
            os.unlink(filename) 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def test_preload_resources(self):
        if multiprocessing.get_start_method() != 'forkserver':
            self.skipTest("test only relevant for 'forkserver' method")
        name = os.path.join(os.path.dirname(__file__), 'mp_preload.py')
        rc, out, err = test.support.script_helper.assert_python_ok(name)
        out = out.decode()
        err = err.decode()
        if out.rstrip() != 'ok' or err != '':
            print(out)
            print(err)
            self.fail("failed spawning forkserver or grandchild")


#
# Check that killing process does not leak named semaphores
# 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def init_processes(addr, port, gpu_num, backend):
    from mpi4py import MPI
    comm = MPI.COMM_WORLD
    size = comm.Get_size()
    rank = comm.Get_rank()
    print(rank, size)
    if mp.get_start_method(allow_none=True) != 'spawn':
        mp.set_start_method('spawn')
    torch.cuda.set_device(rank % gpu_num)
    os.environ['MASTER_ADDR'] = addr
    os.environ['MASTER_PORT'] = port
    os.environ['WORLD_SIZE'] = str(size)
    os.environ['RANK'] = str(rank)
    dist.init_process_group(backend)
    print('initialize {} successfully (rank {})'.format(backend, rank))
    return rank, size 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def __init__(self, pool_names, max_restarts=0, options=None):
		self.names = pool_names
		self.queue = multiprocessing.Queue()
		self.pool = dict()
		self.max_restarts = max_restarts
		self.options = options or dict()

		self.dog_path = os.curdir
		self.dog_handler = LiveReload(self)
		# self.dog_observer = Observer()
		# self.dog_observer.schedule(self.dog_handler, self.dog_path, recursive=True)

		if multiprocessing.get_start_method() != 'fork':  # pragma: no cover
			root_logger = logging.getLogger()
			self.log_listener = QueueListener(self.queue, *root_logger.handlers)

		# TODO: Find out how to get the watchdog + livereload working on a later moment.
		# self.dog_observer.start()

		self._restarts = dict() 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def dist_init(launcher, backend='nccl', **kwargs):
    if mp.get_start_method(allow_none=True) is None:
        mp.set_start_method('spawn')
    if launcher == 'pytorch':
        _init_dist_pytorch(backend, **kwargs)
    elif launcher == 'mpi':
        _init_dist_mpi(backend, **kwargs)
    elif launcher == 'slurm':
        _init_dist_slurm(backend, **kwargs)
    else:
        raise ValueError('Invalid launcher type: {}'.format(launcher)) 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def default_test_processes():
    """Default number of test processes when using the --parallel option."""
    # The current implementation of the parallel test runner requires
    # multiprocessing to start subprocesses with fork().
    if multiprocessing.get_start_method() != 'fork':
        return 1
    try:
        return int(os.environ['DJANGO_TEST_PROCESSES'])
    except KeyError:
        return multiprocessing.cpu_count() 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def _get_multiprocessing_start_method(self):
        """
        Determine method of creating new processes by checking if the
        mp_start_method is set in configs, else, it uses the OS default.
        """
        if conf.has_option('core', 'mp_start_method'):
            return conf.get('core', 'mp_start_method')

        return multiprocessing.get_start_method() 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def dist_init(port):
    if mp.get_start_method(allow_none=True) != 'spawn':
        mp.set_start_method('spawn')
    proc_id = int(os.environ['SLURM_PROCID'])
    ntasks = int(os.environ['SLURM_NTASKS'])
    node_list = os.environ['SLURM_NODELIST']
    num_gpus = torch.cuda.device_count()
    torch.cuda.set_device(proc_id % num_gpus)

    if '[' in node_list:
        beg = node_list.find('[')
        pos1 = node_list.find('-', beg)
        if pos1 < 0:
            pos1 = 1000
        pos2 = node_list.find(',', beg)
        if pos2 < 0:
            pos2 = 1000
        node_list = node_list[:min(pos1, pos2)].replace('[', '')
    addr = node_list[8:].replace('-', '.')
    print(addr)

    os.environ['MASTER_PORT'] = port
    os.environ['MASTER_ADDR'] = addr
    os.environ['WORLD_SIZE'] = str(ntasks)
    os.environ['RANK'] = str(proc_id)
    dist.init_process_group(backend='nccl')

    rank = dist.get_rank()
    world_size = dist.get_world_size()
    return rank, world_size 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def test_noforkbomb(self):
        sm = multiprocessing.get_start_method()
        name = os.path.join(os.path.dirname(__file__), 'mp_fork_bomb.py')
        if sm != 'fork':
            rc, out, err = test.support.script_helper.assert_python_failure(name, sm)
            self.assertEqual(out, b'')
            self.assertIn(b'RuntimeError', err)
        else:
            rc, out, err = test.support.script_helper.assert_python_ok(name, sm)
            self.assertEqual(out.rstrip(), b'123')
            self.assertEqual(err, b'')

#
# Issue #17555: ForkAwareThreadLock
# 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def test_closefd(self):
        if not HAS_REDUCTION:
            raise unittest.SkipTest('requires fd pickling')

        reader, writer = multiprocessing.Pipe()
        fd = self.get_high_socket_fd()
        try:
            p = multiprocessing.Process(target=self._test_closefds,
                                        args=(writer, fd))
            p.start()
            writer.close()
            e = reader.recv()
            p.join(timeout=5)
        finally:
            self.close(fd)
            writer.close()
            reader.close()

        if multiprocessing.get_start_method() == 'fork':
            self.assertIs(e, None)
        else:
            WSAENOTSOCK = 10038
            self.assertIsInstance(e, OSError)
            self.assertTrue(e.errno == errno.EBADF or
                            e.winerror == WSAENOTSOCK, e)

#
# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc
# 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def _check_context(cls, conn):
        conn.send(multiprocessing.get_start_method()) 

# 需要導入模塊: import multiprocessing [as 別名]
# 或者: from multiprocessing import get_start_method [as 別名]
def check_context(self, ctx):
        r, w = ctx.Pipe(duplex=False)
        p = ctx.Process(target=self._check_context, args=(w,))
        p.start()
        w.close()
        child_method = r.recv()
        r.close()
        p.join()
        self.assertEqual(child_method, ctx.get_start_method())