Python bz2.decompress方法代码示例

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def get_data():
		'''
		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                    plot
		filename                 subjectivity_html/obj/2002/Abandon.html
		text           A senior at an elite college (Katie Holmes), a...
		movie_name                                               abandon
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def get_full_data():
		'''
		Returns all plots and reviews, not just the ones that appear in movies with both plot descriptions and reviews.

		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                             plot
		text                    Vijay Singh Rajput (Amitabh Bachchan) is a qui...
		movie_name                                                        aankhen
		has_plot_and_reviews                                                False
		Name: 0, dtype: object
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus_full.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def decode(self, buf, out=None):

        # normalise inputs
        buf = ensure_contiguous_ndarray(buf)
        if out is not None:
            out = ensure_contiguous_ndarray(out)

        # N.B., bz2 cannot handle ndarray directly because of truth testing issues
        buf = memoryview(buf)

        # do decompression
        dec = _bz2.decompress(buf)

        # handle destination - Python standard library bz2 module does not
        # support direct decompression into buffer, so we have to copy into
        # out if given
        return ndarray_copy(dec, out) 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def _load_file(f, compressor, dtype):
    try:
        data = f.read()
        if not len(data):
            return np.zeros(0, dtype=dtype)

        data = COMPRESSORS[compressor]['decompress'](data)
        try:
            return np.frombuffer(data, dtype=dtype)
        except ValueError as e:
            raise ValueError(f"ValueError while loading data with dtype =\n\t{dtype}") from e   
            
    except Exception:
        raise strax.DataCorrupted(
            f"Fatal Error while reading file {f}: "
            + strax.utils.formatted_exception()) 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def get_tokens(self, text):
        if isinstance(text, text_type):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def test_bad_compression_lib_no_compression(self):
        """Pretend that the zlib/bz2 library compress() method doesn't perform any comnpression"""

        def _mock_get_compress_and_decompress_func(
            compression_type, compression_level=9
        ):
            m = MagicMock()
            # simulate module.compress() method that does not compress input data string
            m.compress = lambda data, compression_level=9: data
            m.decompress = lambda data: data
            return m.compress, m.decompress

        @patch(
            "scalyr_agent.util.get_compress_and_decompress_func",
            new=_mock_get_compress_and_decompress_func,
        )
        def _test(compression_type):
            self.assertIsNone(verify_and_get_compress_func(compression_type))

        _test("deflate")
        _test("bz2")
        _test("lz4")
        _test("zstandard") 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def attributesToBinary(cls, attributes):
        """
        :rtype: (str|None,int)
        :return: the binary data and the number of chunks it was composed from
        """
        chunks = [(int(k), v) for k, v in iteritems(attributes) if cls._isValidChunkName(k)]
        chunks.sort()
        numChunks = int(attributes[u'numChunks'])
        if numChunks:
            if USING_PYTHON2:
                serializedJob = b''.join(v for k, v in chunks)
            else:
                serializedJob = b''.join(v.encode() for k, v in chunks)
            compressed = base64.b64decode(serializedJob)
            if compressed[0] == b'C'[0]:
                binary = bz2.decompress(compressed[1:])
            elif compressed[0] == b'U'[0]:
                binary = compressed[1:]
            else:
                raise RuntimeError('Unexpected prefix {}'.format(compressed[0]))
        else:
            binary = None
        return binary, numChunks 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def test_retrieve_bz2_file_obj(self, boto_mock, tmpdir):
        """
        Test the retrieve_file_obj method with a bz2 file
        """
        # Setup the WAL
        source = tmpdir.join('wal_dir/000000080000ABFF000000C1')
        source.write('something'.encode('utf-8'), ensure=True)
        # Create a simple S3WalUploader obj
        uploader = S3WalUploader(
            mock.MagicMock(), 'test-server', compression='bzip2'
        )
        open_file = uploader.retrieve_file_obj(source.strpath)
        # Check the in memory file received
        assert open_file
        # Decompress on the fly to check content
        assert bz2.decompress(open_file.read()) == 'something'.encode('utf-8') 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def decompress(self, data):
        if six.PY2:
            data = bytes(data)

        if self is CompressionAlgorithm.Uncompressed:
            return data

        if self is CompressionAlgorithm.ZIP:
            return zlib.decompress(data, -15)

        if self is CompressionAlgorithm.ZLIB:
            return zlib.decompress(data)

        if self is CompressionAlgorithm.BZ2:
            return bz2.decompress(data)

        raise NotImplementedError(self) 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def get_tokens(self, text):
        if isinstance(text, str):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def _decompress(data, algorithm):
        """
        Decompress a byte string based of the provided algorithm.
        :param data: byte string
        :param algorithm: string  with the name of the compression algorithm used
        :return: decompressed byte string.
        """
        if algorithm is None or algorithm == 'none':
            result = data
        elif algorithm == 'zlib':
            result = zlib.decompress(data)
        elif algorithm == 'bz2':
            result = bz2.decompress(data)
        else:
            raise ValueError("Compression {} is not supported.".format(algorithm))

        return result 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def test_serialize_dataframe(self):
        """Test serializing a dataframe."""
        df = pd.DataFrame([
            [1, 2, 3],
            [4, 5, 6]
        ], columns=['a', 'b', 'c'])

        result = reports.serialize_dataframe(df)
        self.assertEqual(
            bz2.decompress(result),
            b'\n'.join(
                [
                    b'a,b,c',
                    b'1,2,3',
                    b'4,5,6',
                    b''
                ]
            )
        ) 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def getFile(cls, getfile, unpack=True):
        if cls.getProxy():
            proxy = req.ProxyHandler({'http': cls.getProxy(), 'https': cls.getProxy()})
            auth = req.HTTPBasicAuthHandler()
            opener = req.build_opener(proxy, auth, req.HTTPHandler)
            req.install_opener(opener)
        if cls.ignoreCerts():
            ctx = ssl.create_default_context()
            ctx.check_hostname = False
            ctx.verify_mode = ssl.CERT_NONE
            opener = req.build_opener(urllib.request.HTTPSHandler(context=ctx))
            req.install_opener(opener)

        response = req.urlopen(getfile)
        data = response
        # TODO: if data == text/plain; charset=utf-8, read and decode
        if unpack:
            if   'gzip' in response.info().get('Content-Type'):
                buf = BytesIO(response.read())
                data = gzip.GzipFile(fileobj=buf)
            elif 'bzip2' in response.info().get('Content-Type'):
                data = BytesIO(bz2.decompress(response.read()))
            elif 'zip' in response.info().get('Content-Type'):
                fzip = zipfile.ZipFile(BytesIO(response.read()), 'r')
                if len(fzip.namelist())>0:
                    data=BytesIO(fzip.read(fzip.namelist()[0]))
        return (data, response)


    # Feeds 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def bz2_pack(source):
    """
    Returns `source` as bzip2-compressed Python script
    """
    import bz2
    compressed = base64.b64encode(bz2.compress(
        source.encode('utf-8'))).decode('utf-8')
    return f'import bz2,base64;exec(bz2.decompress(base64.b64decode("{compressed}")))' 

# 需要导入模块: import bz2 [as 别名]
# 或者: from bz2 import decompress [as 别名]
def gz_pack(source):
    """
    Returns `source` as gzip-compressed Python script
    """
    import zlib
    compressed = base64.b64encode(zlib.compress(source.encode('utf-8'))).decode('utf-8')
    return f'import zlib,base64;exec(zlib.decompress(base64.b64decode("{compressed}")))'