Python iostream.PipeIOStream類代碼示例

    def call_process(self, cmd, stream, address, io_loop=None): 
        """ Calls process 

        cmd: command in a list e.g ['ls', '-la']
        stdout_callback: callback to run on stdout


        TODO: add some way of calling proc.kill() if the stream is closed
        """

        stdout_stream = Subprocess.STREAM 
        stderr_stream = Subprocess.STREAM 
        proc = Subprocess(cmd, stdout=stdout_stream, stderr=stderr_stream)
        call_back = partial(self.on_exit, address)
        proc.set_exit_callback(call_back)

        pipe_stream = PipeIOStream(proc.stdout.fileno())

        try:
            while True:
                str_ = yield pipe_stream.read_bytes(102400, partial=True)
                yield stream.write(str_)
        except StreamClosedError:
            pass
        print("end address: {}".format(address))

    def __init__(self, stdin=sys.stdin, stdout=sys.stdout, context: dict={}):
        """
        Create a new shell.

        :param stdin: file handle of the stdandard input
        :param stdout: file handle of the standard output
        :param context: exposed variables
        """
        self.stdin = PipeIOStream(stdin.fileno())
        self.stdout = PipeIOStream(stdout.fileno())
        self.input_buffer = []
        self.running = False
        self.context = context

class WriteFileHandler(web.RequestHandler):
    @web.asynchronous
    def get(self):
        print 'handler begin at %s' % datetime.now()
        self.f = open('test.data', 'w')
        self.stream = PipeIOStream(self.f.fileno())
        self.stream.write(test_data, self.callback)
        print 'handler async write at %s' % datetime.now()

    def callback(self):
        self.f.close()
        self.finish()
        print 'handler complete at %s' % datetime.now()

    def test_pipe_iostream_big_write(self):
        r, w = os.pipe()

        rs = PipeIOStream(r, io_loop=self.io_loop)
        ws = PipeIOStream(w, io_loop=self.io_loop)

        NUM_BYTES = 1048576

        # Write 1MB of data, which should fill the buffer
        ws.write(b"1" * NUM_BYTES)

        rs.read_bytes(NUM_BYTES, self.stop)
        data = self.wait()
        self.assertEqual(data, b"1" * NUM_BYTES)

        ws.close()
        rs.close()

    def __init__(self, rpipe, wpipe=None, auto_close=False):
        """Pipe-based stream

        NOTE: reading from or writing to files, use os.open to get the file
        descriptor instead of python's open. Socket file descriptors and
        others are fine.

        when you use os.pipe to generate one write pipe and one read pipe, you
        need to pass both of them into init method.

        :param rpipe: an integer file descriptor which supports read ops
        :param wpipe: an integer file descriptor which supports write ops
        :param auto: flag to indicate to close the stream automatically or not
        """
        assert rpipe is not None
        self._rpipe = rpipe
        self._wpipe = wpipe

        self._rs = PipeIOStream(self._rpipe) if self._rpipe is not None else None
        self._ws = PipeIOStream(self._wpipe) if self._wpipe is not None else None
        self.auto_close = auto_close
        self.state = StreamState.init

        self.exception = None

class Shell(object):
    def __init__(self, stdin=sys.stdin, stdout=sys.stdout, context: dict={}):
        """
        Create a new shell.

        :param stdin: file handle of the stdandard input
        :param stdout: file handle of the standard output
        :param context: exposed variables
        """
        self.stdin = PipeIOStream(stdin.fileno())
        self.stdout = PipeIOStream(stdout.fileno())
        self.input_buffer = []
        self.running = False
        self.context = context

    def start(self):
        self.running = True
        self.stdout.write(b"\r$>")
        self.stdin.read_until(b'\n', self.on_line)

    def on_line(self, chunk_bytes: bytes):
        chunk = chunk_bytes.decode('utf-8', errors='ignore').rstrip('\n')
        if not chunk.endswith('\\'):
            self.input_buffer.append(chunk.strip())
            line = " ".join(self.input_buffer)
            self.input_buffer.clear()
            self.on_command(line)
        else:
            self.input_buffer.append(chunk[:-1].strip())
            self.stdout.write(b"\r  ")
        if self.running:
            self.start()

    @tornado.gen.engine
    def on_command(self, command):
        try:
            if command:
                code = compile(command + '\n', '<shell>', 'single')
                res = eval(code, self.context)
                if res is not None:
                    r = pprint.pformat(res).encode('utf-8')
                    yield tornado.gen.Task(self.stdout.write, r + b'\n')
        except SystemExit:
            raise
        except:
            yield tornado.gen.Task(self.stdout.write, traceback.format_exc().encode('utf-8'))

    def test_pipe_iostream(self):
        r, w = os.pipe()

        rs = PipeIOStream(r, io_loop=self.io_loop)
        ws = PipeIOStream(w, io_loop=self.io_loop)

        ws.write(b"hel")
        ws.write(b"lo world")

        rs.read_until(b" ", callback=self.stop)
        data = self.wait()
        self.assertEqual(data, b"hello ")

        rs.read_bytes(3, self.stop)
        data = self.wait()
        self.assertEqual(data, b"wor")

        ws.close()

        rs.read_until_close(self.stop)
        data = self.wait()
        self.assertEqual(data, b"ld")

        rs.close()

 def get(self):
     print 'handler begin at %s' % datetime.now()
     self.f = open('test.data', 'w')
     self.stream = PipeIOStream(self.f.fileno())
     self.stream.write(test_data, self.callback)
     print 'handler async write at %s' % datetime.now()

    def __init__(
                self,
                command,
                timeout=-1,
                stdout_chunk_callback=None,
                stderr_chunk_callback=None,
                exit_process_callback=None,
                stdin_bytes=None,
                io_loop=None,
                kill_on_timeout=False
                ):
        """
        Initializes the subprocess with callbacks and timeout.

        :param command: command like ['java', '-jar', 'test.jar']
        :param timeout: timeout for subprocess to complete, if negative or \
        zero then no timeout
        :param stdout_chunk_callback: callback(bytes_data_chuck_from_stdout)
        :param stderr_chunk_callback: callback(bytes_data_chuck_from_stderr)
        :param exit_process_callback: callback(exit_code, \
        was_expired_by_timeout)
        :param stdin_bytes: bytes data to send to stdin
        :param io_loop: tornado io loop on None for current
        :param kill_on_timeout: kill(-9) or terminate(-15)?
        """
        self.aa_exit_process_callback = exit_process_callback
        self.aa_kill_on_timeout = kill_on_timeout
        stdin = Subprocess.STREAM if stdin_bytes else None
        stdout = Subprocess.STREAM if stdout_chunk_callback else None
        stderr = Subprocess.STREAM if stderr_chunk_callback else None

        Subprocess.__init__(self, command, stdin=stdin, stdout=stdout,
                            stderr=stderr, io_loop=io_loop)

        self.aa_process_expired = False
        self.aa_terminate_timeout = self.io_loop.call_later(
            timeout, self.aa_timeout_callback) if timeout > 0 else None

        self.set_exit_callback(self.aa_exit_callback)

        if stdin:
            self.stdin.write(stdin_bytes)
            self.stdin.close()

        if stdout:
            output_stream = PipeIOStream(self.stdout.fileno())

            def on_stdout_chunk(data):
                stdout_chunk_callback(data)
                if not output_stream.closed():
                    output_stream.read_bytes(102400,
                                             on_stdout_chunk, None, True)

            output_stream.read_bytes(102400, on_stdout_chunk, None, True)

        if stderr:
            stderr_stream = PipeIOStream(self.stderr.fileno())

            def on_stderr_chunk(data):
                stderr_chunk_callback(data)
                if not stderr_stream.closed():
                    stderr_stream.read_bytes(102400,
                                             on_stderr_chunk, None, True)

            stderr_stream.read_bytes(102400, on_stderr_chunk, None, True)

class PipeStream(Stream):
    def __init__(self, rpipe, wpipe=None, auto_close=False):
        """Pipe-based stream

        NOTE: reading from or writing to files, use os.open to get the file
        descriptor instead of python's open. Socket file descriptors and
        others are fine.

        when you use os.pipe to generate one write pipe and one read pipe, you
        need to pass both of them into init method.

        :param rpipe: an integer file descriptor which supports read ops
        :param wpipe: an integer file descriptor which supports write ops
        :param auto: flag to indicate to close the stream automatically or not
        """
        assert rpipe is not None
        self._rpipe = rpipe
        self._wpipe = wpipe

        self._rs = PipeIOStream(self._rpipe) if self._rpipe is not None else None
        self._ws = PipeIOStream(self._wpipe) if self._wpipe is not None else None
        self.auto_close = auto_close
        self.state = StreamState.init

        self.exception = None

    @tornado.gen.coroutine
    def read(self):
        if self.exception:
            raise self.exception

        if self.state == StreamState.completed or self._rpipe is None:
            raise tornado.gen.Return("")
        elif self.state == StreamState.init:
            self.state = StreamState.streaming

        chunk = ""
        try:
            chunk = yield self._rs.read_bytes(common.MAX_PAYLOAD_SIZE, partial=True)

        except StreamClosedError:
            # reach the end of the pipe stream
            self.state = StreamState.completed
        finally:
            if self.exception:
                raise self.exception
            raise tornado.gen.Return(chunk)

    @tornado.gen.coroutine
    def write(self, chunk):
        assert self._wpipe is not None
        if self.exception:
            raise self.exception

        try:
            yield self._ws.write(chunk)
            self.state = StreamState.streaming
        except StreamClosedError:
            self.state = StreamState.completed
            raise UnexpectedError("Stream has been closed.")
        finally:
            if self.exception:
                raise self.exception

    def set_exception(self, exception):
        self.exception = exception
        self.close()

    def close(self):
        self.state = StreamState.completed
        if self._ws and self.auto_close:
            self._ws.close()

        if self._rs and self.auto_close:
            self._rs.close()