Python gzip.GzipFile方法代码示例

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def ls(args):
    bucket = resources.s3.Bucket(args.billing_reports_bucket.format(account_id=ARN.get_account_id()))
    now = datetime.utcnow()
    year = args.year or now.year
    month = str(args.month or now.month).zfill(2)
    next_year = year + ((args.month or now.month) + 1) // 12
    next_month = str(((args.month or now.month) + 1) % 12).zfill(2)
    manifest_name = "aegea/{report}/{yr}{mo}01-{next_yr}{next_mo}01/{report}-Manifest.json"
    manifest_name = manifest_name.format(report=__name__, yr=year, mo=month, next_yr=next_year, next_mo=next_month)
    try:
        manifest = json.loads(bucket.Object(manifest_name).get().get("Body").read())
        for report_key in manifest["reportKeys"]:
            report = BytesIO(bucket.Object(report_key).get().get("Body").read())
            with gzip.GzipFile(fileobj=report) as fh:
                reader = csv.DictReader(fh)
                for line in reader:
                    page_output(tabulate(filter_line_items(reader, args), args))
    except ClientError as e:
        msg = 'Unable to get report {} from {}: {}. Run "aegea billing configure" to enable reports.'
        raise AegeaException(msg.format(manifest_name, bucket, e)) 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def test_write_dataflow_header(self):
    pipeline = TestPipeline()
    pcoll = pipeline | 'Create' >> beam.Create(self.variants, reshuffle=False)
    headers = ['foo\n']
    _ = pcoll | 'Write' >> vcfio.WriteToVcf(
        self.path + '.gz',
        compression_type=CompressionTypes.AUTO,
        headers=headers)
    pipeline.run()

    read_result = []
    for file_name in glob.glob(self.path + '*'):
      with gzip.GzipFile(file_name, 'r') as f:
        read_result.extend(f.read().splitlines())

    self.assertEqual(read_result[0], 'foo')
    for actual, expected in zip(read_result[1:], self.variant_lines):
      self._assert_variant_lines_equal(actual, expected) 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def gzip_encode(data):
    """data -> gzip encoded data

    Encode data using the gzip content encoding as described in RFC 1952
    """
    if not gzip:
        raise NotImplementedError
    f = BytesIO()
    gzf = gzip.GzipFile(mode="wb", fileobj=f, compresslevel=1)
    gzf.write(data)
    gzf.close()
    encoded = f.getvalue()
    f.close()
    return encoded

##
# Decode a string using the gzip content encoding such as specified by the
# Content-Encoding: gzip
# in the HTTP header, as described in RFC 1952
#
# @param data The encoded data
# @return the unencoded data
# @raises ValueError if data is not correctly coded. 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def gzip_decode(data):
    """gzip encoded data -> unencoded data

    Decode data using the gzip content encoding as described in RFC 1952
    """
    if not gzip:
        raise NotImplementedError
    f = BytesIO(data)
    gzf = gzip.GzipFile(mode="rb", fileobj=f)
    try:
        decoded = gzf.read()
    except IOError:
        raise ValueError("invalid data")
    f.close()
    gzf.close()
    return decoded

##
# Return a decoded file-like object for the gzip encoding
# as described in RFC 1952.
#
# @param response A stream supporting a read() method
# @return a file-like object that the decoded data can be read() from 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def decompress(cls, data):
        '''Decompress gzip-compressed data `data`.

        It will perform basic validation, then return the decompressed
        data or raises ValueError exception for invalid `data`.

        :param data: Gzip-compressed data to decompress.
        :type data: ``bytes``
        :returns: decompressed data.
        :rtype: ``string``

        :raises ValueError: If `data` is not in gzip format
        '''

        if not cls.check_format(data):
            raise ValueError('File is not gzip format.')

        return gzip.GzipFile(fileobj=BytesIO(data),
                             mode='rb').read() 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def _decompressContent(response, new_content):
    content = new_content
    try:
        encoding = response.get("content-encoding", None)
        if encoding in ["gzip", "deflate"]:
            if encoding == "gzip":
                content = gzip.GzipFile(fileobj=io.BytesIO(new_content)).read()
            if encoding == "deflate":
                content = zlib.decompress(content, -zlib.MAX_WBITS)
            response["content-length"] = str(len(content))
            # Record the historical presence of the encoding in a way the won't interfere.
            response["-content-encoding"] = response["content-encoding"]
            del response["content-encoding"]
    except (IOError, zlib.error):
        content = ""
        raise FailedToDecompressContent(
            _("Content purported to be compressed with %s but failed to decompress.")
            % response.get("content-encoding"),
            response,
            content,
        )
    return content 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def _decompressContent(response, new_content):
    content = new_content
    try:
        encoding = response.get("content-encoding", None)
        if encoding in ["gzip", "deflate"]:
            if encoding == "gzip":
                content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read()
            if encoding == "deflate":
                content = zlib.decompress(content, -zlib.MAX_WBITS)
            response["content-length"] = str(len(content))
            # Record the historical presence of the encoding in a way the won't interfere.
            response["-content-encoding"] = response["content-encoding"]
            del response["content-encoding"]
    except (IOError, zlib.error):
        content = ""
        raise FailedToDecompressContent(
            _("Content purported to be compressed with %s but failed to decompress.")
            % response.get("content-encoding"),
            response,
            content,
        )
    return content 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def compress_file_with_gzip(file_name, tmp_dir):
        """Compresses a file with GZIP.

        Args:
            file_name: Local path to file to be compressed.
            tmp_dir: Temporary directory where an GZIP file will be created.

        Returns:
            A tuple of gzip file name and size.
        """
        logger = getLogger(__name__)
        base_name = os.path.basename(file_name)
        gzip_file_name = os.path.join(tmp_dir, base_name + '_c.gz')
        logger.debug('gzip file: %s, original file: %s', gzip_file_name,
                     file_name)
        fr = open(file_name, 'rb')
        fw = gzip.GzipFile(gzip_file_name, 'wb')
        shutil.copyfileobj(fr, fw)
        fw.close()
        fr.close()
        SnowflakeFileUtil.normalize_gzip_header(gzip_file_name)

        statinfo = os.stat(gzip_file_name)
        return gzip_file_name, statinfo.st_size 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def handleResponse(self, data):
        if (self.isCompressed):
            logging.debug("Decompressing content...")
            data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(data)).read()
            
        logging.log(self.getLogLevel(), "Read from server:\n" + data)
        #logging.log(self.getLogLevel(), "Read from server:\n <large data>" )


        data = self.replaceSecureLinks(data)

        if (self.contentLength != None):
            self.client.setHeader('Content-Length', len(data))
        
        self.client.write(data)
        self.shutdown() 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def retrieve_url_nodecode(url):
    """ Return the content of the url page as a string """
    req = Request(url, headers=headers)
    try:
        response = urlopen(req)
    except URLError as errno:
        print(" ".join(("Connection error:", str(errno.reason))))
        print(" ".join(("URL:", url)))
        return ""
    dat = response.read()
    # Check if it is gzipped
    if dat[:2] == '\037\213':
        # Data is gzip encoded, decode it
        compressedstream = StringIO(dat)
        gzipper = gzip.GzipFile(fileobj=compressedstream)
        extracted_data = gzipper.read()
        dat = extracted_data
        return dat
    return dat 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def test_gzip_loadtxt():
    # Thanks to another windows brokenness, we can't use
    # NamedTemporaryFile: a file created from this function cannot be
    # reopened by another open call. So we first put the gzipped string
    # of the test reference array, write it to a securely opened file,
    # which is then read from by the loadtxt function
    s = BytesIO()
    g = gzip.GzipFile(fileobj=s, mode='w')
    g.write(b'1 2 3\n')
    g.close()

    s.seek(0)
    with temppath(suffix='.gz') as name:
        with open(name, 'wb') as f:
            f.write(s.read())
        res = np.loadtxt(name)
    s.close()

    assert_array_equal(res, [1, 2, 3]) 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def write_to_file(self, save_file):
    "Write all the times to file."
    try:
      with TestTimes.LockedFile(save_file, 'a+b') as fd:
        times = TestTimes.__read_test_times_file(fd)

        if times is None:
          times = self.__times
        else:
          times.update(self.__times)

        # We erase data from file while still holding a lock to it. This
        # way reading old test times and appending new ones are atomic
        # for external viewer.
        fd.seek(0)
        fd.truncate()
        with gzip.GzipFile(fileobj=fd, mode='wb') as gzf:
          cPickle.dump(times, gzf, PICKLE_HIGHEST_PROTOCOL)
    except IOError:
      pass  # ignore errors---saving the times isn't that important 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def parse_content_encoding(self, response_headers, response_data):
        """
        Parses a response that contains Content-Encoding to retrieve
        response_data
        """
        if response_headers['content-encoding'] == 'gzip':
            buf = StringIO.StringIO(response_data)
            zipbuf = gzip.GzipFile(fileobj=buf)
            response_data = zipbuf.read()
        elif response_headers['content-encoding'] == 'deflate':
            data = StringIO.StringIO(zlib.decompress(response_data))
            response_data = data.read()
        else:
            raise errors.TestError(
                'Received unknown Content-Encoding',
                {
                    'content-encoding':
                        str(response_headers['content-encoding']),
                    'function': 'http.HttpResponse.parse_content_encoding'
                })
        return response_data 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def encode_dockerfile(args):
    with io.BytesIO() as buf:
        with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
            gz.write(get_dockerfile(args))
            gz.close()
        return base64.b64encode(buf.getvalue()).decode() 

# 需要导入模块: import gzip [as 别名]
# 或者: from gzip import GzipFile [as 别名]
def gzip_compress_bytes(payload):
    buf = io.BytesIO()
    with gzip.GzipFile(fileobj=buf, mode="w", mtime=0) as gzfh:
        gzfh.write(payload)
    return buf.getvalue()