Python BufferedWriter.flush方法代码示例

# 需要导入模块: from io import BufferedWriter [as 别名]
# 或者: from io.BufferedWriter import flush [as 别名]
def consecutive_download(node_id: str, file: io.BufferedWriter, **kwargs):
    """Keyword args: write_callback"""
    r = BackOffRequest.get(get_content_url() + 'nodes/' + node_id + '/content', stream=True)
    if r.status_code not in OK_CODES:
        raise RequestError(r.status_code, r.text)

    write_callback = kwargs.get('write_callback', None)

    total_ln = int(r.headers.get('content-length'))
    length = kwargs.get('length', None)
    if length and total_ln != length:
        logging.info('Length mismatch: argument %d, content %d' % (length, total_ln))

    pgo = progress.Progress()
    curr_ln = 0
    try:
        for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ):
            if chunk:  # filter out keep-alive new chunks
                file.write(chunk)
                file.flush()
                if write_callback:
                    write_callback(chunk)
                curr_ln += len(chunk)
                pgo.print_progress(total_ln, curr_ln)
    except (ConnectionError, ReadTimeoutError) as e:
        raise RequestError(RequestError.CODE.READ_TIMEOUT, '[acd_cli] Timeout. ' + e.__str__())
    print()  # break progress line
    r.close()
    return

# 需要导入模块: from io import BufferedWriter [as 别名]
# 或者: from io.BufferedWriter import flush [as 别名]
    def chunked_download(self, node_id: str, file: io.BufferedWriter, **kwargs):
        """:param kwargs:
        offset (int): byte offset -- start byte for ranged request
        length (int): total file length[!], equal to end + 1
        write_callbacks (list[function])
        """
        ok_codes = [http.PARTIAL_CONTENT]

        write_callbacks = kwargs.get("write_callbacks", [])

        chunk_start = kwargs.get("offset", 0)
        length = kwargs.get("length", 100 * 1024 ** 4)

        retries = 0
        while chunk_start < length:
            chunk_end = chunk_start + CHUNK_SIZE - 1
            if chunk_end >= length:
                chunk_end = length - 1

            if retries >= CHUNK_MAX_RETRY:
                raise RequestError(
                    RequestError.CODE.FAILED_SUBREQUEST, "[acd_api] Downloading chunk failed multiple times."
                )
            r = self.BOReq.get(
                self.content_url + "nodes/" + node_id + "/content",
                stream=True,
                acc_codes=ok_codes,
                headers={"Range": "bytes=%d-%d" % (chunk_start, chunk_end)},
            )

            logger.debug("Range %d-%d" % (chunk_start, chunk_end))
            # this should only happen at the end of unknown-length downloads
            if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE:
                logger.debug("Invalid byte range requested %d-%d" % (chunk_start, chunk_end))
                break
            if r.status_code not in ok_codes:
                r.close()
                retries += 1
                logging.debug("Chunk [%d-%d], retry %d." % (chunk_start, chunk_end, retries))
                continue

            curr_ln = 0
            try:
                for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ):
                    if chunk:  # filter out keep-alive new chunks
                        file.write(chunk)
                        file.flush()
                        for wcb in write_callbacks:
                            wcb(chunk)
                        curr_ln += len(chunk)
            finally:
                r.close()

            chunk_start += CHUNK_SIZE
            retries = 0

        return

# 需要导入模块: from io import BufferedWriter [as 别名]
# 或者: from io.BufferedWriter import flush [as 别名]
def chunked_download(node_id: str, file: io.BufferedWriter, **kwargs):
    """Keyword args:
    offset: byte offset
    length: total length, equal to end - 1
    write_callback
    """
    ok_codes = [http.PARTIAL_CONTENT]

    write_callback = kwargs.get('write_callback', None)

    length = kwargs.get('length', 100 * 1024 ** 4)

    pgo = progress.Progress()
    chunk_start = kwargs.get('offset', 0)
    retries = 0
    while chunk_start < length:
        chunk_end = chunk_start + CHUNK_SIZE - 1
        if chunk_end >= length:
            chunk_end = length - 1

        if retries >= CHUNK_MAX_RETRY:
            raise RequestError(RequestError.CODE.FAILED_SUBREQUEST,
                               '[acd_cli] Downloading chunk failed multiple times.')
        r = BackOffRequest.get(get_content_url() + 'nodes/' + node_id + '/content', stream=True,
                               acc_codes=ok_codes,
                               headers={'Range': 'bytes=%d-%d' % (chunk_start, chunk_end)})

        logger.debug('Range %d-%d' % (chunk_start, chunk_end))
        # this should only happen at the end of unknown-length downloads
        if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE:
            logger.debug('Invalid byte range requested %d-%d' % (chunk_start, chunk_end))
            break
        if r.status_code not in ok_codes:
            r.close()
            retries += 1
            logging.debug('Chunk [%d-%d], retry %d.' % (retries, chunk_start, chunk_end))
            continue

        try:
            curr_ln = 0
            for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ):
                if chunk:  # filter out keep-alive new chunks
                    file.write(chunk)
                    file.flush()
                    if write_callback:
                        write_callback(chunk)
                    curr_ln += len(chunk)
                    pgo.print_progress(length, curr_ln + chunk_start)
            chunk_start += CHUNK_SIZE
            retries = 0
            r.close()
        except (ConnectionError, ReadTimeoutError) as e:
            file.close()
            raise RequestError(RequestError.CODE.READ_TIMEOUT, '[acd_cli] Timeout. ' + e.__str__())

    print()  # break progress line
    return

# 需要导入模块: from io import BufferedWriter [as 别名]
# 或者: from io.BufferedWriter import flush [as 别名]
    def chunked_download(self, node_id: str, file: io.BufferedWriter, **kwargs):
        """:param kwargs:
        offset (int): byte offset -- start byte for ranged request
        length (int): total file length[!], equal to end + 1
        write_callbacks (list[function])
        """
        ok_codes = [http.PARTIAL_CONTENT]

        write_callbacks = kwargs.get('write_callbacks', [])

        chunk_start = kwargs.get('offset', 0)
        length = kwargs.get('length', 100 * 1024 ** 4)

        dl_chunk_sz = self._conf.getint('transfer', 'dl_chunk_size')

        retries = 0
        while chunk_start < length:
            chunk_end = chunk_start + dl_chunk_sz - 1
            if chunk_end >= length:
                chunk_end = length - 1

            if retries >= self._conf.getint('transfer', 'chunk_retries'):
                raise RequestError(RequestError.CODE.FAILED_SUBREQUEST,
                                   '[acd_api] Downloading chunk failed multiple times.')
            r = self.BOReq.get(self.content_url + 'nodes/' + node_id + '/content', stream=True,
                               acc_codes=ok_codes,
                               headers={'Range': 'bytes=%d-%d' % (chunk_start, chunk_end)})

            logger.debug('Node "%s", range %d-%d' % (node_id, chunk_start, chunk_end))
            # this should only happen at the end of unknown-length downloads
            if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE:
                r.close()
                logger.debug('Invalid byte range requested %d-%d' % (chunk_start, chunk_end))
                break
            if r.status_code not in ok_codes:
                r.close()
                retries += 1
                logging.debug('Chunk [%d-%d], retry %d.' % (chunk_start, chunk_end, retries))
                continue

            curr_ln = 0
            try:
                for chunk in r.iter_content(chunk_size=self._conf.getint('transfer', 'fs_chunk_size')):
                    if chunk:  # filter out keep-alive new chunks
                        file.write(chunk)
                        file.flush()
                        for wcb in write_callbacks:
                            wcb(chunk)
                        curr_ln += len(chunk)
            finally:
                r.close()
                chunk_start = file.tell()

            retries = 0

        return

# 需要导入模块: from io import BufferedWriter [as 别名]
# 或者: from io.BufferedWriter import flush [as 别名]
def chunked_download(node_id: str, file: io.BufferedWriter, **kwargs):
    """Keyword args:
    offset (int): byte offset -- start byte for ranged request
    length (int): total file length[!], equal to end + 1
    write_callbacks: (list[function])
    """
    ok_codes = [http.PARTIAL_CONTENT]

    write_callbacks = kwargs.get('write_callbacks', [])

    chunk_start = kwargs.get('offset', 0)
    length = kwargs.get('length', 100 * 1024 ** 4)

    retries = 0
    while chunk_start < length:
        chunk_end = chunk_start + CHUNK_SIZE - 1
        if chunk_end >= length:
            chunk_end = length - 1

        if retries >= CHUNK_MAX_RETRY:
            raise RequestError(RequestError.CODE.FAILED_SUBREQUEST,
                               '[acd_cli] Downloading chunk failed multiple times.')
        r = BackOffRequest.get(get_content_url() + 'nodes/' + node_id + '/content', stream=True,
                               acc_codes=ok_codes,
                               headers={'Range': 'bytes=%d-%d' % (chunk_start, chunk_end)})

        logger.debug('Range %d-%d' % (chunk_start, chunk_end))
        # this should only happen at the end of unknown-length downloads
        if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE:
            logger.debug('Invalid byte range requested %d-%d' % (chunk_start, chunk_end))
            break
        if r.status_code not in ok_codes:
            r.close()
            retries += 1
            logging.debug('Chunk [%d-%d], retry %d.' % (chunk_start, chunk_end, retries))
            continue

        curr_ln = 0
        # connection exceptions occur here
        for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ):
            if chunk:  # filter out keep-alive new chunks
                file.write(chunk)
                file.flush()
                for wcb in write_callbacks:
                    wcb(chunk)
                curr_ln += len(chunk)
        chunk_start += CHUNK_SIZE
        retries = 0
        r.close()

    return

# 需要导入模块: from io import BufferedWriter [as 别名]
# 或者: from io.BufferedWriter import flush [as 别名]
def write(data, wfile: io.BufferedWriter) -> None:
    wfile.write(json.dumps(data).encode() + b"\n")
    wfile.flush()

# 需要导入模块: from io import BufferedWriter [as 别名]
# 或者: from io.BufferedWriter import flush [as 别名]

#.........这里部分代码省略.........
        """Writes a floating point value (4 bytes)"""
        self.writer.write(pack('<f', value))
        self.written_count += 4

    def write_double(self, value):
        """Writes a floating point value (8 bytes)"""
        self.writer.write(pack('<d', value))
        self.written_count += 8

    def write_large_int(self, value, bits, signed=True):
        """Writes a n-bits long integer value"""
        self.writer.write(
            int.to_bytes(
                value, length=bits // 8, byteorder='little', signed=signed))
        self.written_count += bits // 8

    def write(self, data):
        """Writes the given bytes array"""
        self.writer.write(data)
        self.written_count += len(data)

    # endregion

    # region Telegram custom writing

    def tgwrite_bytes(self, data):
        """Write bytes by using Telegram guidelines"""
        if len(data) < 254:
            padding = (len(data) + 1) % 4
            if padding != 0:
                padding = 4 - padding

            self.write(bytes([len(data)]))
            self.write(data)

        else:
            padding = len(data) % 4
            if padding != 0:
                padding = 4 - padding

            self.write(bytes([254]))
            self.write(bytes([len(data) % 256]))
            self.write(bytes([(len(data) >> 8) % 256]))
            self.write(bytes([(len(data) >> 16) % 256]))
            self.write(data)

        self.write(bytes(padding))

    def tgwrite_string(self, string):
        """Write a string by using Telegram guidelines"""
        self.tgwrite_bytes(string.encode('utf-8'))

    def tgwrite_bool(self, boolean):
        """Write a boolean value by using Telegram guidelines"""
        #                     boolTrue                boolFalse
        self.write_int(0x997275b5 if boolean else 0xbc799737, signed=False)

    def tgwrite_date(self, datetime):
        """Converts a Python datetime object into Unix time (used by Telegram) and writes it"""
        value = 0 if datetime is None else int(datetime.timestamp())
        self.write_int(value)

    def tgwrite_object(self, tlobject):
        """Writes a Telegram object"""
        tlobject.on_send(self)

    def tgwrite_vector(self, vector):
        """Writes a vector of Telegram objects"""
        self.write_int(0x1cb5c415, signed=False)  # Vector's constructor ID
        self.write_int(len(vector))
        for item in vector:
            self.tgwrite_object(item)

    # endregion

    def flush(self):
        """Flush the current stream to "update" changes"""
        self.writer.flush()

    def close(self):
        """Close the current stream"""
        self.writer.close()

    def get_bytes(self, flush=True):
        """Get the current bytes array content from the buffer, optionally flushing first"""
        if flush:
            self.writer.flush()
        return self.writer.raw.getvalue()

    def get_written_bytes_count(self):
        """Gets the count of bytes written in the buffer.
           This may NOT be equal to the stream length if one was provided when initializing the writer"""
        return self.written_count

    # with block
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()