Python BufferedReader.peek方法代码示例

# 需要导入模块: from io import BufferedReader [as 别名]
# 或者: from io.BufferedReader import peek [as 别名]
 def __enter__(self):
     if self.__to_close is not None:
         raise Exception(f"{self!r} is already a context manager")
     stream = IOWrapper(self.__stream)
     reader = BufferedReader(stream.__enter__())
     to_close = [reader]
     if reader.peek(len(GZIP_MAGIC)) == GZIP_MAGIC:
         ret = GzipIOWrapper(reader)
         to_close.append(ret)
         ret = ret.__enter__()
     else:
         ret = reader
     self.__to_close = (stream,) + tuple(to_close)
     return ret

# 需要导入模块: from io import BufferedReader [as 别名]
# 或者: from io.BufferedReader import peek [as 别名]
    
    args = parser.parse_args()
    
    embeddings = []
    
    if args.convert_to == 'txt':  # then format must be bin
        with FileIO(args.filename, 'rb') as f:
            reader = BufferedReader(f)
            vocab_size, dimension = map(int, f.readline().split())
            for _ in range(vocab_size):
                w = ''.join(takewhile(lambda x: x != ' ', (reader.read(1) for _ in count())))
                s = reader.read(4 * dimension)
                reader.read(1)  # end of line character
                arr = np.fromstring(s, dtype=np.float32)
                embeddings.append((w, arr))
            assert not reader.peek(1)
    else:
        with open(args.filename) as f:
            vocab_size, dimension = map(int, f.readline().split())
            for line in f:
                w, s = line.strip().split(' ', 1)
                arr = np.fromstring(s, dtype=np.float32, sep=' ')
                embeddings.append((w, arr))
            assert len(embeddings) == vocab_size

    if args.convert_to == 'txt':
        with open(args.output, 'w') as f:
            f.write('{0} {1}\n'.format(vocab_size, dimension))
            f.writelines('{0} {1} \n'.format(w, ' '.join(map(str, vec))) for w, vec in embeddings)
    else:
        with open(args.output, 'wb') as f:

# 需要导入模块: from io import BufferedReader [as 别名]
# 或者: from io.BufferedReader import peek [as 别名]
class Unpacker:
    def __init__(self, initial_bytes=None):
        self.initial_bytes = initial_bytes
        self.buf = BufferedReader(raw=BytesIO(initial_bytes))

    def struct_unpack(self, fmt):
        return struct.unpack(fmt, self.buf.read(struct.calcsize(fmt)))

    def read_byte(self):
        return self.struct_unpack('B')[0]

    def read_signed_byte(self):
        return self.struct_unpack('b')[0]

    def read_short(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix in [Prefix.INT16, Prefix.UINT16], 'read_short expects prefix %s or %s but read %s' % (
            Prefix.INT16, Prefix.UINT16, prefix)
        return self.struct_unpack('>H' if prefix == Prefix.UINT16 else '>h')[0]

    def read_int(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix in [Prefix.INT32, Prefix.UINT32], 'read_int expects prefix %s or %s but read %s' % (
            Prefix.INT32, Prefix.UINT32, prefix)
        return self.struct_unpack('>I' if prefix == Prefix.UINT32 else '>i')[0]

    def read_long(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix in [Prefix.INT64, Prefix.UINT64], 'read_long expects prefix %s or %s but read %s' % (
            Prefix.INT64, Prefix.UINT64, prefix)
        return self.struct_unpack('>Q' if prefix == Prefix.UINT64 else '>q')[0]

    def read_float(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix == Prefix.FLOAT32, 'read_float expects prefix %s but read %s' % (Prefix.FLOAT32, prefix)
        return self.struct_unpack('>f')[0]

    def read_double(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix == Prefix.FLOAT64, 'read_double expects prefix %s but read %s' % (Prefix.FLOAT64, prefix)
        return self.struct_unpack('>d')[0]

    def read_payload(self, length):
        return self.buf.read(length)

    def peek(self):
        return struct.unpack('B', self.buf.peek()[0])[0]

    def unpack_nil(self):
        assert self.read_byte() == Prefix.NIL, 'unpack_nil expects to read %s' % Prefix.NIL

    def unpack_boolean(self):
        prefix = self.read_byte()
        assert prefix in [Prefix.TRUE, Prefix.FALSE], 'unpack_boolean expects prefix %s or %s but read %s' % (
            Prefix.TRUE, Prefix.FALSE, prefix)
        return prefix == Prefix.TRUE

    def unpack_int(self):
        prefix = self.peek()

        if Prefix.is_fix_int(prefix):
            return self.read_byte() if Prefix.is_pos_fix_int(prefix) else self.read_signed_byte()

        if prefix == Prefix.INT8:
            self.read_byte()
            return self.read_signed_byte()
        elif prefix == Prefix.UINT8:
            self.read_byte()
            return self.read_byte()
        elif prefix in [Prefix.UINT16, Prefix.INT16]:
            return self.read_short()
        elif prefix in [Prefix.UINT32, Prefix.INT32]:
            return self.read_int()
        elif prefix in [Prefix.UINT64, Prefix.INT64]:
            return self.read_long()
        else:
            raise ValueError('unexpected int prefix %s' % prefix)

    def unpack_float(self):
        return self.read_float()

    def unpack_double(self):
        return self.read_double()

    def unpack(self):
        prefix = self.peek()

        if Prefix.is_fix_int(prefix) or prefix in [Prefix.INT8, Prefix.UINT8, Prefix.INT16, Prefix.UINT16, Prefix.INT32,
                                                   Prefix.UINT32, Prefix.INT64, Prefix.UINT64]:
            return self.unpack_int()
        elif Prefix.is_fixed_array(prefix) or prefix in [Prefix.ARRAY16, Prefix.ARRAY32]:
            return self.unpack_array()
        elif Prefix.is_fixed_map(prefix) or prefix in [Prefix.MAP16, Prefix.MAP32]:
            return self.unpack_map()
        elif Prefix.is_fix_str(prefix) or prefix in [Prefix.STR8, Prefix.STR16, Prefix.STR32]:
#.........这里部分代码省略.........

# 需要导入模块: from io import BufferedReader [as 别名]
# 或者: from io.BufferedReader import peek [as 别名]
class MultipartReader(object):
    """
    Reads a stream formatted as multipart/form-data.  Provides each part as a
    readable interface, avoiding loading the entire body into memory or
    caching to a file.

    Usage:

    reader = MultipartReader(f, boundary)
    part1 = reader.next_part()
    print part1.form_name()
    data = part1.read(1024)
    """

    def __init__(self, stream, boundary=None):
        b = "\r\n--" + boundary + "--"
        stream = _StreamWrapper(stream)

        self.buf_reader = BufferedReader(stream)
        self.nl = b[:2]
        self.nl_dash_boundary = b[:len(b)-2]
        self.dash_boundary_dash = b[2:]
        self.dash_boundary = b[2:len(b)-2]
        self.headers = {}
        self.parts_read = 0

        self.current_part = None

    def iter_parts(self):
        """
        Returns an iterator over the Parts in multipart/form-data.

        Do not use if you're skipping the end of the data, as the last
        iteration will seek to the end of the stream.
        """
        part = self.next_part()
        while part != None:
            yield part
            part = self.next_part()

    def next_part(self):
        """
        Returns the next Part in the stream.  If a previous part was not read
        completely, it will seek to the beginning of the next part, closing the
        previous one.
        """

        if self.current_part != None:
            self.current_part.close()

        expect_new_part = False
        while True:
            line = self.buf_reader.readline()
            is_EOF = self.buf_reader.peek(1)
            if len(is_EOF) == 0 and self.is_final_boundary(line):
                return None

            if self.is_boundary_delimeter_line(line):
                #print "Creating new part"
                self.parts_read += 1
                bp = _new_part(self)
                self.current_part = bp
                return bp

            if self.is_final_boundary(line):
                return None

            if expect_new_part:
                raise Exception("expecting a new Part, got line %s" % line)

            if self.parts_read == 0:
                continue

            if line == self.nl:
                expect_new_part = True
                continue

            raise Exception("Unexpected line in next_part(): %s" % line)

    def is_final_boundary(self, line):
        if not line.startswith(self.dash_boundary_dash):
            return False
        rest = line[len(self.dash_boundary_dash):]
        rest = skipLWSPChar(rest)
        return len(rest) == 0 or rest == self.nl

    def is_boundary_delimeter_line(self, line):
        if not line.startswith(self.dash_boundary):
            return False

        rest = line[len(self.dash_boundary):]
        rest = skipLWSPChar(rest)
        if self.parts_read == 0 and len(rest) == 1 and rest[0] == "\n":
            self.nl = self.nl[1:]
            self.nl_dash_boundary = self.nl_dash_boundary[1:]
        return rest == self.nl

    def peek_buffer_is_empty_part(self, peek):
        if peek.startswith(self.dash_boundary_dash):
            rest = peek[len(self.dash_boundary_dash):]
#.........这里部分代码省略.........

# 需要导入模块: from io import BufferedReader [as 别名]
# 或者: from io.BufferedReader import peek [as 别名]

#.........这里部分代码省略.........

        Registered message callback functions will be called with MSGs
        once parsed.

        PONG will be called upon a ping.

        An exception will be raised upon an ERR from gnatsd.

        An +OK doesn't do anything.
        """
        if self.remaining_bytes:
            data = self.remaining_bytes + data
            self.remaining_bytes = b""

        data_buf = BufferedReader(BytesIO(data))
        while True:
            command = data_buf.read(4)
            if command == b"-ERR":
                raise NatsError(data_buf.read())
            elif command == b"+OK\r":
                val = data_buf.read(1)
                if val != b"\n":
                    self.remaining_bytes += command
                    break
            elif command == b"MSG ":
                val = data_buf.readline()
                if not val:
                    self.remaining_bytes += command
                    break
                if not val.endswith(b"\r\n"):
                    self.remaining_bytes += command + val
                    break

                meta_data = val.split(b" ")
                n_bytes = int(meta_data[-1])
                subject = meta_data[0].decode()
                if len(meta_data) == 4:
                    reply_to = meta_data[2].decode()
                elif len(meta_data) == 3:
                    reply_to = None
                else:
                    self.remaining_bytes += command + val
                    break

                sid = meta_data[1].decode()

                if sid in self.sids:
                    on_msg = self.sids[sid]
                else:
                    on_msg = self.on_msg

                payload = data_buf.read(n_bytes)
                if len(payload) != n_bytes:
                    self.remaining_bytes += command + val + payload
                    break

                if on_msg:
                    on_msg(nats_protocol=self, sid=sid, subject=subject, reply_to=reply_to, payload=payload)
                else:
                    stdout.write(command.decode())
                    stdout.write(val.decode())
                    stdout.write(payload.decode())

                payload_post = data_buf.readline()
                if payload_post != b"\r\n":
                    self.remaining_bytes += command + val + payload + payload_post
                    break
            elif command == b"PING":
                self.log.info("got PING")
                self.pong()
                val = data_buf.readline()
                if val != b"\r\n":
                    self.remaining_bytes += command + val
                    break
            elif command == b"PONG":
                self.pout -= 1
                val = data_buf.readline()
                if val != b"\r\n":
                    self.remaining_bytes += command + val
                    break
            elif command == b"INFO":
                val = data_buf.readline()
                if not val.endswith(b"\r\n"):
                    self.remaining_bytes += command + val
                    break
                settings = json.loads(val.decode("utf8"))
                self.server_settings = ServerInfo(**settings)
                self.log.info("{server_info}", server_info=settings)
                self.status = CONNECTED
                self.connect()
                if self.on_connect_d:
                    self.on_connect_d.callback(self)
                    self.on_connect_d = None
            else:
                self.log.info(b"Not handled command is: {command!r}", command=command)
                val = data_buf.read()
                self.remaining_bytes += command + val
            if not data_buf.peek(1):
                self.log.debug("emptied data")
                break