Python io.BufferedReader类代码示例

def read(rfile: io.BufferedReader) -> typing.Any:
    x = rfile.readline().strip()
    return json.loads(x)

    def load(cls, filename, metadata_only=False):
        """
        Load ring data from a file.

        :param filename: Path to a file serialized by the save() method.
        :param bool metadata_only: If True, only load `devs` and `part_shift`.
        :returns: A RingData instance containing the loaded data.
        """
        gz_file = BufferedReader(GzipFile(filename, 'rb'))

        # See if the file is in the new format
        magic = gz_file.read(4)
        if magic == b'R1NG':
            format_version, = struct.unpack('!H', gz_file.read(2))
            if format_version == 1:
                ring_data = cls.deserialize_v1(
                    gz_file, metadata_only=metadata_only)
            else:
                raise Exception('Unknown ring format version %d' %
                                format_version)
        else:
            # Assume old-style pickled ring
            gz_file.seek(0)
            ring_data = pickle.load(gz_file)

        if not hasattr(ring_data, 'devs'):
            ring_data = RingData(ring_data['replica2part2dev_id'],
                                 ring_data['devs'], ring_data['part_shift'],
                                 ring_data.get('next_part_power'))
        return ring_data

    def test_io_bufferedreader(self):
        fp = BytesIO(b'foo')
        resp = HTTPResponse(fp, preload_content=False)
        br = BufferedReader(resp)

        self.assertEqual(br.read(), b'foo')

        br.close()
        self.assertEqual(resp.closed, True)

    def tst_io_bufferedreader(self):

        fp = self._fake_fp(b'foo')
        #fp = BytesIO(b'foo')
        resp = HTTPResponse(fp, preload_content=False)
        br = BufferedReader(resp)

        self.assertEqual(br.read(), b'foo')

        br.close()
        self.assertEqual(resp.closed, True)

        b = b'fooandahalf'
        fp = self._fake_fp(b)
        #fp = BytesIO(b)
        resp = HTTPResponse(fp, preload_content=False)
        br = BufferedReader(resp, 5)

        br.read(1)  # sets up the buffer, reading 5
        self.assertEqual(len(fp.read()), len(b) - 5)

        # This is necessary to make sure the "no bytes left" part of `readinto`
        # gets tested.
        while not br.closed:
            br.read(5)

 def __enter__(self):
     if self.__to_close is not None:
         raise Exception(f"{self!r} is already a context manager")
     stream = IOWrapper(self.__stream)
     reader = BufferedReader(stream.__enter__())
     to_close = [reader]
     if reader.peek(len(GZIP_MAGIC)) == GZIP_MAGIC:
         ret = GzipIOWrapper(reader)
         to_close.append(ret)
         ret = ret.__enter__()
     else:
         ret = reader
     self.__to_close = (stream,) + tuple(to_close)
     return ret

    def __init__(self, data=None, stream=None):
        if data:
            self.stream = BytesIO(data)
        elif stream:
            self.stream = stream
        else:
            raise ValueError('Either bytes or a stream must be provided')

        self.reader = BufferedReader(self.stream)
        self._last = None  # Should come in handy to spot -404 errors

    def __init__(self, data=None, stream=None):
        if data:
            self.stream = BytesIO(data)
        elif stream:
            self.stream = stream
        else:
            raise InvalidParameterError(
                'Either bytes or a stream must be provided')

        self.reader = BufferedReader(self.stream)

    def __init__(self, stream, boundary=None):
        b = "\r\n--" + boundary + "--"
        stream = _StreamWrapper(stream)

        self.buf_reader = BufferedReader(stream)
        self.nl = b[:2]
        self.nl_dash_boundary = b[:len(b)-2]
        self.dash_boundary_dash = b[2:]
        self.dash_boundary = b[2:len(b)-2]
        self.headers = {}
        self.parts_read = 0

        self.current_part = None

def _wait_for_line_in_stream(stream: io.BufferedReader, timeout: float) -> str:
    """Wait for a line to appear in a stream and return it.

    This will only work on Unix.
    If something does appear in the stream, but it isn't terminated with a newline, then this
    function will hang. But since the program that we will use this for will write its output in
    lines, I don't think that the additional robustness is needed.
    """
    line_selector = select.poll()
    line_selector.register(stream, select.POLLIN)

    start_time = time.perf_counter()
    while time.perf_counter() - start_time < timeout:
        if line_selector.poll(0.01):
            return stream.readline()
    pytest.fail('Waiting for a stream line timed out.')

    def test_io_bufferedreader(self):
        fp = BytesIO(b'foo')
        resp = HTTPResponse(fp, preload_content=False)
        br = BufferedReader(resp)

        assert br.read() == b'foo'

        br.close()
        assert resp.closed

        b = b'fooandahalf'
        fp = BytesIO(b)
        resp = HTTPResponse(fp, preload_content=False)
        br = BufferedReader(resp, 5)

        br.read(1)  # sets up the buffer, reading 5
        assert len(fp.read()) == (len(b) - 5)

        # This is necessary to make sure the "no bytes left" part of `readinto`
        # gets tested.
        while not br.closed:
            br.read(5)

    def read_line(self):
        if not self.file:
            path = self.get_path()
            if not path:
                raise Exception('Unable to find the location of "Plex Media Server.log"')

            # Open file
            self.file = ASIO.open(path, opener=False)
            self.file.seek(self.file.get_size(), SEEK_ORIGIN_CURRENT)

            # Create buffered reader
            self.reader = BufferedReader(self.file)

            self.path = self.file.get_path()
            log.info('Opened file path: "%s"' % self.path)

        return self.reader.readline()

    def read(self, file_path):
        while not self.writing:
            time.sleep(1)

        print "Read starting..."

        f = ASIO.open(file_path, opener=False)
        s = BufferedReader(f)

        orig_path = f.get_path()
        stale_since = None

        while True:
            if f is None:
                print 'Opening file...'
                f = ASIO.open(file_path, opener=False)
                s = BufferedReader(f)

            # Try read line
            line = s.readline()

            if line:
                stale_since = None
                time.sleep(0.05)
            else:
                if stale_since is None:
                    stale_since = time.time()
                    time.sleep(0.1)
                    continue
                elif (time.time() - stale_since) > 2 and f.get_path() != orig_path:
                    s.close()
                    s = None

                    f.close()
                    f = None
                elif not self.writing:
                    break
                else:
                    time.sleep(0.1)
                    continue

            print 'read %r' % (line,)

        print 'finished'
        s.close()
        f.close()

class Server:
    """
    classdocs
    """

    def __init__(self):
        """
        Constructor
        """
        # Read the configuration parameters from tulsi.conf
        self.timestr = time.strftime("%Y:%m:%d-%H:%M:%S")
        logging.basicConfig(filename="/var/log/tulsi.log", level=logging.DEBUG)
        try:
            self.conf = ConfigParser.ConfigParser()
            self.conf.read("/etc/tulsi/tulsi.conf")
            self.udp_ip = self.conf.get("tulsi", "host")
            self.udp_port = int(self.conf.get("tulsi", "port"))

            # printing the host and port of tulsi

            logging.info("%s The IP of the host: %s" % (self.timestr, self.udp_ip))
            logging.info("%s The  Port number of the host :%s" % (self.timestr, self.udp_port))
        except:
            # Error message of tulsi not working
            logging.error("The tulsi configuration file is not found")

            # Creating objects of MessageEncode and HostInfo
        msg_encode = MessageEncode()
        host_info = HostInfo()

        # Initializing empty lists
        self.drives = []
        self.service = []
        self.ip_array = []
        self.ring_ip = []
        self.ring_conf_ip = []
        self.ring_drives = []
        self.ip_set_array = []
        self.my_ring_conf = dict()
        # Read the ring Configuration file
        self.gz_file = GzipFile("/etc/swift/container.ring.gz", "rb")
        if hasattr(self.gz_file, "_checkReadable"):
            self.gz_file = BufferedReader(self.gz_file)
        magic = self.gz_file.read(4)
        if magic == "R1NG":
            version, = struct.unpack("!H", self.gz_file.read(2))
            if version == 1:
                self.ring_data = self.read_ring_file(self.gz_file)
            else:
                logging.error("%s Unknown ring format version %d" % (self.timestr, version))
                raise Exception("Unknown ring format version %d" % version)

        # While loop to continuously check the status of  swift services and
        # drives and send information to tulsi client
        while True:
            self.ip_array = host_info.read_ip()
            self.service = host_info.read_services()
            self.drives = host_info.read_drives(self.drives)
            self.message = msg_encode.create_message(
                self.my_ring_conf, self.ring_conf_ip, self.ip_array, self.service, self.drives
            )
            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # Internet  # UDP
            sock.sendto(self.message, (self.udp_ip, self.udp_port))
            time.sleep(5)
            self.ip_array = []
            self.service = []
            self.drives = []

    #  Function to extract the ip  and devices from container.ring.gz
    def read_ring_file(self, sgz_file):
        self.json_len, = struct.unpack("!I", self.gz_file.read(4))
        self.ring_dict = json.loads(self.gz_file.read(self.json_len))
        logging.info("Extracted Ring data : %s", self.ring_dict)
        self.ring_dict["replica2part2dev_id"] = []
        self.partition_count = 1 << (32 - self.ring_dict["part_shift"])
        # logging.info('%s The Ip from ring file %s' % (self.timestr, self.ring_conf_ip[]]))
        # logging.info('%s The IP of host machine %s' %(self.timestr, self.ip_array))
        for x in self.ring_dict["devs"]:
            self.mystring = x
            if self.mystring["ip"] in self.my_ring_conf:
                # append the new number to the existing array at this slot
                self.my_ring_conf[self.mystring["ip"]].append(self.mystring["device"])
            else:
                # create a new array in this slot
                self.my_ring_conf[self.mystring["ip"]] = [self.mystring["device"]]
                self.ring_conf_ip.append(self.mystring["ip"])
                logging.info("%s The Ip from ring file %s" % (self.timestr, self.ring_conf_ip))
                logging.info("%s The IP of host machine %s" % (self.timestr, self.ip_array))

class Unpacker:
    def __init__(self, initial_bytes=None):
        self.initial_bytes = initial_bytes
        self.buf = BufferedReader(raw=BytesIO(initial_bytes))

    def struct_unpack(self, fmt):
        return struct.unpack(fmt, self.buf.read(struct.calcsize(fmt)))

    def read_byte(self):
        return self.struct_unpack('B')[0]

    def read_signed_byte(self):
        return self.struct_unpack('b')[0]

    def read_short(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix in [Prefix.INT16, Prefix.UINT16], 'read_short expects prefix %s or %s but read %s' % (
            Prefix.INT16, Prefix.UINT16, prefix)
        return self.struct_unpack('>H' if prefix == Prefix.UINT16 else '>h')[0]

    def read_int(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix in [Prefix.INT32, Prefix.UINT32], 'read_int expects prefix %s or %s but read %s' % (
            Prefix.INT32, Prefix.UINT32, prefix)
        return self.struct_unpack('>I' if prefix == Prefix.UINT32 else '>i')[0]

    def read_long(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix in [Prefix.INT64, Prefix.UINT64], 'read_long expects prefix %s or %s but read %s' % (
            Prefix.INT64, Prefix.UINT64, prefix)
        return self.struct_unpack('>Q' if prefix == Prefix.UINT64 else '>q')[0]

    def read_float(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix == Prefix.FLOAT32, 'read_float expects prefix %s but read %s' % (Prefix.FLOAT32, prefix)
        return self.struct_unpack('>f')[0]

    def read_double(self, prefix=None):
        if prefix is None:
            prefix = self.read_byte()
        assert prefix == Prefix.FLOAT64, 'read_double expects prefix %s but read %s' % (Prefix.FLOAT64, prefix)
        return self.struct_unpack('>d')[0]

    def read_payload(self, length):
        return self.buf.read(length)

    def peek(self):
        return struct.unpack('B', self.buf.peek()[0])[0]

    def unpack_nil(self):
        assert self.read_byte() == Prefix.NIL, 'unpack_nil expects to read %s' % Prefix.NIL

    def unpack_boolean(self):
        prefix = self.read_byte()
        assert prefix in [Prefix.TRUE, Prefix.FALSE], 'unpack_boolean expects prefix %s or %s but read %s' % (
            Prefix.TRUE, Prefix.FALSE, prefix)
        return prefix == Prefix.TRUE

    def unpack_int(self):
        prefix = self.peek()

        if Prefix.is_fix_int(prefix):
            return self.read_byte() if Prefix.is_pos_fix_int(prefix) else self.read_signed_byte()

        if prefix == Prefix.INT8:
            self.read_byte()
            return self.read_signed_byte()
        elif prefix == Prefix.UINT8:
            self.read_byte()
            return self.read_byte()
        elif prefix in [Prefix.UINT16, Prefix.INT16]:
            return self.read_short()
        elif prefix in [Prefix.UINT32, Prefix.INT32]:
            return self.read_int()
        elif prefix in [Prefix.UINT64, Prefix.INT64]:
            return self.read_long()
        else:
            raise ValueError('unexpected int prefix %s' % prefix)

    def unpack_float(self):
        return self.read_float()

    def unpack_double(self):
        return self.read_double()

    def unpack(self):
        prefix = self.peek()

        if Prefix.is_fix_int(prefix) or prefix in [Prefix.INT8, Prefix.UINT8, Prefix.INT16, Prefix.UINT16, Prefix.INT32,
                                                   Prefix.UINT32, Prefix.INT64, Prefix.UINT64]:
            return self.unpack_int()
        elif Prefix.is_fixed_array(prefix) or prefix in [Prefix.ARRAY16, Prefix.ARRAY32]:
            return self.unpack_array()
        elif Prefix.is_fixed_map(prefix) or prefix in [Prefix.MAP16, Prefix.MAP32]:
            return self.unpack_map()
        elif Prefix.is_fix_str(prefix) or prefix in [Prefix.STR8, Prefix.STR16, Prefix.STR32]:
#.........这里部分代码省略.........

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=help_msg)

    parser.add_argument('filename', help='input file')
    parser.add_argument('output', help='output file')
    parser.add_argument('convert_to', choices=['bin', 'txt'])
    
    args = parser.parse_args()
    
    embeddings = []
    
    if args.convert_to == 'txt':  # then format must be bin
        with FileIO(args.filename, 'rb') as f:
            reader = BufferedReader(f)
            vocab_size, dimension = map(int, f.readline().split())
            for _ in range(vocab_size):
                w = ''.join(takewhile(lambda x: x != ' ', (reader.read(1) for _ in count())))
                s = reader.read(4 * dimension)
                reader.read(1)  # end of line character
                arr = np.fromstring(s, dtype=np.float32)
                embeddings.append((w, arr))
            assert not reader.peek(1)
    else:
        with open(args.filename) as f:
            vocab_size, dimension = map(int, f.readline().split())
            for line in f:
                w, s = line.strip().split(' ', 1)
                arr = np.fromstring(s, dtype=np.float32, sep=' ')
                embeddings.append((w, arr))