Python multiprocessing.Pipe方法代码示例

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def __init__(self, env_fns, spaces=None):
        """
        envs: list of gym environments to run in subprocesses
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
            for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def __init__(self, constructor):
    """Step environment in a separate process for lock free paralellism.

    The environment will be created in the external process by calling the
    specified callable. This can be an environment class, or a function
    creating the environment and potentially wrapping it. The returned
    environment should not access global variables.

    Args:
      constructor: Callable that creates and returns an OpenAI gym environment.

    Attributes:
      observation_space: The cached observation space of the environment.
      action_space: The cached action space of the environment.
    """
    self._conn, conn = multiprocessing.Pipe()
    self._process = multiprocessing.Process(
        target=self._worker, args=(constructor, conn))
    atexit.register(self.close)
    self._process.start()
    self._observ_space = None
    self._action_space = None 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def __init__(self, env_fns, render_interval):
        """ Minor addition to SubprocVecEnv, automatically renders environments

        envs: list of gym environments to run in subprocesses
        """
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
            for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        self.action_space, self.observation_space = self.remotes[0].recv()

        self.render_interval = render_interval
        self.render_timer = 0 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def __init__(self, env_fns):
        if np.__version__ == '1.16.0':
            warnings.warn("""
NumPy 1.16.0 can cause severe memory leak in chainerrl.envs.MultiprocessVectorEnv.
We recommend using other versions of NumPy.
See https://github.com/numpy/numpy/issues/12793 for details.
""")  # NOQA

        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = \
            [Process(target=worker, args=(work_remote, env_fn))
             for (work_remote, env_fn) in zip(self.work_remotes, env_fns)]
        for p in self.ps:
            p.start()
        self.last_obs = [None] * self.num_envs
        self.remotes[0].send(('get_spaces', None))
        self.action_space, self.observation_space = self.remotes[0].recv()
        self.closed = False 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def _pipe(self):
        """On Windows we use a pipe to emulate a Linux style character
        buffer."""
        if self._evdev:
            return None

        if not self.__pipe:
            target_function = self._get_target_function()
            if not target_function:
                return None

            self.__pipe, child_conn = Pipe(duplex=False)
            self._listener = Process(target=target_function,
                                     args=(child_conn,), daemon=True)
            self._listener.start()
        return self.__pipe 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def __init__(self, env_fns, spaces=None):
        """
        Arguments:

        env_fns: iterable of callables -  functions that create environments to run in subprocesses. Need to be cloud-pickleable
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
                   for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True  # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        self.viewer = None
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def tcp_server_piped(port=19543, use_stderr_logger=False):
    """
    TCP server context-manager used for integration tests.

    It starts server on localhost/given-port during context-manager start()
    and performs server-cleanup during context-manager stop()

    :param port: where to start server at
    :param use_stderr_logger: configure logging to output into stderr?
    :return: pair (server, inter-process-pipe used to control server)
    """
    client_process_pipe_endpoint, server_pipe_endpoint = Pipe()
    tcp_server = TcpServerPiped(host='localhost', port=port,
                                pipe_in=server_pipe_endpoint, delay=0,
                                use_stderr_logger=use_stderr_logger)
    tcp_server.start()
    time.sleep(0.5)  # allow server to boot-up
    # returning tcp_server let's you call tcp_server.terminate() to kill server
    # for testing dropped TCP connection
    yield (tcp_server, client_process_pipe_endpoint)
    client_process_pipe_endpoint.send(("shutdown", {}))
    tcp_server.join()

# ------------------- TCP server running in separate process --------------------- 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def main():
    parent_conn, child_conn = Pipe()

    child = Process(target=work, args=(child_conn,))

    for item in (
        42,
        'some string',
        {'one': 1},
        CustomClass(),
        None,
    ):
        print(
            "PRNT: send: {}".format(item)
        )
        parent_conn.send(item)

    child.start()
    child.join() 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def __init__(self, env_fns, spaces=None):
        """
        envs: list of gym environments to run in subprocesses
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.nenvs = nenvs
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
            for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def __init__(self, env_fns, horizon, trajectories, seed=0):
        """Launch subprocess workers and store configuration parameters.
        :param env_fns (list<()->ResettableEnv>): list of thunks.
        :param horizon (int): length of trajectories to search over.
        :param trajectories (int): minimum number of trajectories to evaluate.
               It will be rounded up to the nearest multiple of len(make_env)."""
        super().__init__(horizon, trajectories)
        nremotes = len(env_fns)
        # Integer ceiling of self.trajectories / nworkers
        traj_per_worker = (self.trajectories - 1) // nremotes + 1

        pipes = [Pipe() for _ in range(nremotes)]
        self.remotes, self.work_remotes = zip(*pipes)
        worker_cfgs = zip(self.work_remotes, self.remotes, env_fns)
        self.ps = []
        for i, (work_remote, remote, dynamic_fn) in enumerate(worker_cfgs):
            args = (work_remote, remote, CloudpickleWrapper(dynamic_fn), horizon, traj_per_worker)
            process = Process(target=_worker, args=args)
            process.daemon = True
            # If the main process crashes, we should not cause things to hang
            process.start()
            self.ps.append(process)
        for remote in self.work_remotes:
            remote.close() 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def run(engine_factory, source_name, input_queue_maxsize, port, num_tokens,
        message_max_size=None):
    engine_read, server_write = multiprocessing.Pipe(duplex=False)

    # We cannot read from multiprocessing.Pipe without blocking the event
    # loop
    server_read, engine_write = os.pipe()

    local_server = _LocalServer(
        num_tokens, input_queue_maxsize, server_write, server_read)
    local_server.add_source_consumed(source_name)

    engine_process = multiprocessing.Process(
        target=_run_engine,
        args=(engine_factory, engine_read, server_read, engine_write))
    try:
        engine_process.start()
        os.close(engine_write)
        local_server.launch(port, message_max_size)
    finally:
        local_server.cleanup()
        os.close(server_read)

    raise Exception('Server stopped') 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def test_recursion(self):
        rconn, wconn = self.Pipe(duplex=False)
        self._test_recursion(wconn, [])

        time.sleep(DELTA)
        result = []
        while rconn.poll():
            result.append(rconn.recv())

        expected = [
            [],
              [0],
                [0, 0],
                [0, 1],
              [1],
                [1, 0],
                [1, 1]
            ]
        self.assertEqual(result, expected) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def test_spawn_close(self):
        # We test that a pipe connection can be closed by parent
        # process immediately after child is spawned.  On Windows this
        # would have sometimes failed on old versions because
        # child_conn would be closed before the child got a chance to
        # duplicate it.
        conn, child_conn = self.Pipe()

        p = self.Process(target=self._echo, args=(child_conn,))
        p.daemon = True
        p.start()
        child_conn.close()    # this might complete before child initializes

        msg = latin('hello')
        conn.send_bytes(msg)
        self.assertEqual(conn.recv_bytes(), msg)

        conn.send_bytes(SENTINEL)
        conn.close()
        p.join() 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def test_fd_transfer(self):
        if self.TYPE != 'processes':
            self.skipTest("only makes sense with processes")
        conn, child_conn = self.Pipe(duplex=True)

        p = self.Process(target=self._writefd, args=(child_conn, b"foo"))
        p.daemon = True
        p.start()
        self.addCleanup(support.unlink, support.TESTFN)
        with open(support.TESTFN, "wb") as f:
            fd = f.fileno()
            if msvcrt:
                fd = msvcrt.get_osfhandle(fd)
            reduction.send_handle(conn, fd, p.pid)
        p.join()
        with open(support.TESTFN, "rb") as f:
            self.assertEqual(f.read(), b"foo") 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import Pipe [as 别名]
def test_large_fd_transfer(self):
        # With fd > 256 (issue #11657)
        if self.TYPE != 'processes':
            self.skipTest("only makes sense with processes")
        conn, child_conn = self.Pipe(duplex=True)

        p = self.Process(target=self._writefd, args=(child_conn, b"bar", True))
        p.daemon = True
        p.start()
        self.addCleanup(support.unlink, support.TESTFN)
        with open(support.TESTFN, "wb") as f:
            fd = f.fileno()
            for newfd in range(256, MAXFD):
                if not self._is_fd_assigned(newfd):
                    break
            else:
                self.fail("could not find an unassigned large file descriptor")
            os.dup2(fd, newfd)
            try:
                reduction.send_handle(conn, newfd, p.pid)
            finally:
                os.close(newfd)
        p.join()
        with open(support.TESTFN, "rb") as f:
            self.assertEqual(f.read(), b"bar")