Python zstandard.ZstdDecompressor方法代码示例

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def decompress(data, compressor_id):
    if compressor_id == SnappyContext.compressor_id:
        # python-snappy doesn't support the buffer interface.
        # https://github.com/andrix/python-snappy/issues/65
        # This only matters when data is a memoryview since
        # id(bytes(data)) == id(data) when data is a bytes.
        # NOTE: bytes(memoryview) returns the memoryview repr
        # in Python 2.7. The right thing to do in 2.7 is call
        # memoryview.tobytes(), but we currently only use
        # memoryview in Python 3.x.
        return snappy.uncompress(bytes(data))
    elif compressor_id == ZlibContext.compressor_id:
        return zlib.decompress(data)
    elif compressor_id == ZstdContext.compressor_id:
        # ZstdDecompressor is not thread safe.
        # TODO: Use a pool?
        return ZstdDecompressor().decompress(data)
    else:
        raise ValueError("Unknown compressorId %d" % (compressor_id,)) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def _read_rel_zs_rows(filepath, chunk_size=8 * 1000 * 1000):
    from zstandard import ZstdDecompressor

    with open(filepath, "rb") as fh:
        ctx = ZstdDecompressor()
        with ctx.stream_reader(fh) as reader:
            over = False
            chunks = []
            rows = []
            while not over:
                have_row = False
                while not have_row:
                    chunk = reader.read(chunk_size)
                    if not chunk:
                        over = True
                        break
                    if b"\n" in chunk:
                        have_row = True
                    chunks.append(chunk)
                (new_rows, semi_row) = _consume_rows(chunks)
                rows += new_rows
                chunks = [semi_row]
    return rows 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def open_tar_zst(path, mode):
    if mode == "w":
        cctx = zstandard.ZstdCompressor(threads=-1)
        with open(path, "wb") as f:
            with cctx.stream_writer(f) as compressor:
                with tarfile.open(mode="w|", fileobj=compressor) as tar:
                    yield tar
    elif mode == "r":
        dctx = zstandard.ZstdDecompressor()
        with open(path, "rb") as f:
            with dctx.stream_reader(f) as reader:
                with tarfile.open(mode="r|", fileobj=reader) as tar:
                    yield tar
    else:
        assert False, f"Unexpected mode: {mode}"


# Using tar directly is twice as fast than through Python! 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def data_file(self):
        """Return the uncompressed raw CPIO data of the RPM archive."""

        if self._data_file is None:
            fileobj = _SubFile(self._fileobj, self.data_offset)

            if self.headers["archive_compression"] == b"xz":
                if not getattr(sys.modules[__name__], "lzma", False):
                    raise NoLZMAModuleError("lzma module not present")
                self._data_file = lzma.LZMAFile(fileobj)
            elif self.headers["archive_compression"] == b"zstd":
                if not getattr(sys.modules[__name__], "zstandard", False):
                    raise NoZSTANDARDModuleError("zstandard module not present")
                if not (sys.version_info.major >= 3 and sys.version_info.minor >= 5):
                    raise NoBytesIOError("Need io.BytesIO (Python >= 3.5)")
                with zstandard.ZstdDecompressor().stream_reader(fileobj) as zstd_data:
                    self._data_file = io.BytesIO(zstd_data.read())
            else:
                self._data_file = gzip.GzipFile(fileobj=fileobj)

        return self._data_file 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def decompress(self, source, cursor, compressedbytes, uncompressedbytes=None):
        if self.algo == uproot.const.kZLIB:
            from zlib import decompress as zlib_decompress
            return zlib_decompress(cursor.bytes(source, compressedbytes))

        elif self.algo == uproot.const.kLZMA:
            try:
                from lzma import decompress as lzma_decompress
            except ImportError:
                try:
                    from backports.lzma import decompress as lzma_decompress
                except ImportError:
                    raise ImportError("install lzma package with:\n    pip install backports.lzma\nor\n    conda install backports.lzma\n(or just use Python >= 3.3).")
            return lzma_decompress(cursor.bytes(source, compressedbytes))

        elif self.algo == uproot.const.kOldCompressionAlgo:
            raise NotImplementedError("ROOT's \"old\" algorithm (fCompress 300) is not supported")

        elif self.algo == uproot.const.kLZ4:
            try:
                from lz4.block import decompress as lz4_decompress
            except ImportError:
                raise ImportError("install lz4 package with:\n    pip install lz4\nor\n    conda install lz4")

            if uncompressedbytes is None:
                raise ValueError("lz4 needs to know the uncompressed number of bytes")
            return lz4_decompress(cursor.bytes(source, compressedbytes), uncompressed_size=uncompressedbytes)

        elif self.algo == uproot.const.kZSTD:
            try:
                import zstandard as zstd
            except ImportError:
                raise ImportError("install zstd package with:\n    pip install zstandard\nor\n    conda install zstandard")
            dctx = zstd.ZstdDecompressor()
            return dctx.decompress(cursor.bytes(source, compressedbytes))

        else:
            raise ValueError("unrecognized compression algorithm: {0}".format(self.algo)) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def zstandard_file(infile, mode="rb"):
        if "r" in mode:
            cctx = zstd.ZstdDecompressor()
            return cctx.stream_reader(infile)
        else:
            cctx = zstd.ZstdCompressor(level=10)
            return cctx.stream_writer(infile) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def test_zstandard(self):
        data = self._data
        compress = verify_and_get_compress_func("zstandard")
        self.assertIsNotNone(compress)

        import zstandard

        decompressor = zstandard.ZstdDecompressor()

        self.assertEqual(data, decompressor.decompress(compress(data))) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def zstd_decompress(path):
    dctx = zstandard.ZstdDecompressor()
    with open(f"{path}.zst", "rb") as input_f:
        with open(path, "wb") as output_f:
            dctx.copy_stream(input_f, output_f) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def _db_open(path, mode):
    parts = str(path).split(".")
    assert len(parts) > 1, "Extension needed to figure out serialization format"
    if len(parts) == 2:
        db_format = parts[-1]
        compression = None
    else:
        db_format = parts[-2]
        compression = parts[-1]

    assert compression is None or compression in COMPRESSION_FORMATS
    assert db_format in SERIALIZATION_FORMATS

    store_constructor = SERIALIZATION_FORMATS[db_format]

    if compression == "gz":
        with gzip.GzipFile(path, mode) as f:
            yield store_constructor(f)
    elif compression == "zstd":
        if "w" in mode or "a" in mode:
            cctx = zstandard.ZstdCompressor()
            with open(path, mode) as f:
                with cctx.stream_writer(f) as writer:
                    yield store_constructor(writer)
        else:
            dctx = zstandard.ZstdDecompressor()
            with open(path, mode) as f:
                with dctx.stream_reader(f) as reader:
                    yield store_constructor(reader)
    else:
        with open(path, mode) as f:
            yield store_constructor(f) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def _create_decompressor(self, alg):
        if alg == "snappy":
            return snappy.StreamDecompressor()
        elif alg == "lzma":
            return lzma.LZMADecompressor()
        elif alg == "zstd":
            return zstd.ZstdDecompressor().decompressobj()
        raise InvalidConfigurationError("invalid compression algorithm: {!r}".format(alg)) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def __init__(self, next_fp):
        self._zstd = zstd.ZstdDecompressor().decompressobj()
        super().__init__(next_fp)
        self._done = False 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def download_report(self, report):
        """
        Download and extract a json+zstd covdir report
        """
        assert isinstance(report, Report)

        # Check the report is available on remote storage
        blob = self.bucket.blob(report.gcp_path)
        if not blob.exists():
            logger.debug("No report found on GCP", path=report.gcp_path)
            return False

        if os.path.exists(report.path):
            logger.info("Report already available", path=report.path)
            return True

        os.makedirs(os.path.dirname(report.archive_path), exist_ok=True)
        blob.download_to_filename(report.archive_path)
        logger.info("Downloaded report archive", path=report.archive_path)

        with open(report.path, "wb") as output:
            with open(report.archive_path, "rb") as archive:
                dctx = zstd.ZstdDecompressor()
                reader = dctx.stream_reader(archive)
                while True:
                    chunk = reader.read(16384)
                    if not chunk:
                        break
                    output.write(chunk)

        os.unlink(report.archive_path)
        logger.info("Decompressed report", path=report.path)
        return True 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def itkimage_from_json(js, manager=None):
    """Deserialize a Javascript itk.js Image object."""
    if js is None:
        return None
    else:
        ImageType, dtype = _type_to_image(js['imageType'])
        decompressor = zstd.ZstdDecompressor()
        if six.PY2:
            asBytes = js['compressedData'].tobytes()
            pixelBufferArrayCompressed = np.frombuffer(asBytes, dtype=np.uint8)
        else:
            pixelBufferArrayCompressed = np.frombuffer(js['compressedData'],
                                                       dtype=np.uint8)
        pixelCount = reduce(lambda x, y: x * y, js['size'], 1)
        numberOfBytes = pixelCount * \
            js['imageType']['components'] * np.dtype(dtype).itemsize
        pixelBufferArray = \
            np.frombuffer(decompressor.decompress(pixelBufferArrayCompressed,
                                                  numberOfBytes),
                          dtype=dtype)
        pixelBufferArray.shape = js['size'][::-1]
        # Workaround for GetImageFromArray required until 5.0.1
        # and https://github.com/numpy/numpy/pull/11739
        pixelBufferArrayCopyToBeRemoved = pixelBufferArray.copy()
        # image = itk.PyBuffer[ImageType].GetImageFromArray(pixelBufferArray)
        image = itk.PyBuffer[ImageType].GetImageFromArray(
            pixelBufferArrayCopyToBeRemoved)
        Dimension = image.GetImageDimension()
        image.SetOrigin(js['origin'])
        image.SetSpacing(js['spacing'])
        direction = image.GetDirection()
        directionMatrix = direction.GetVnlMatrix()
        directionJs = js['direction']['data']
        for col in range(Dimension):
            for row in range(Dimension):
                directionMatrix.put(
                    row, col, directionJs[col + row * Dimension])
        return image 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdDecompressor [as 别名]
def decompress_ncz(input,output):
	with open(input, 'rb') as f:
		header = f.read(0x4000)
		magic = readInt64(f)
		sectionCount = readInt64(f)
		sections = []
		for i in range(sectionCount):
			sections.append(Section(f))
			
		dctx = zstandard.ZstdDecompressor()
		reader = dctx.stream_reader(f)
			
		with open(output, 'wb+') as o:
			o.write(header)
			
			while True:
				chunk = reader.read(16384)
				
				if not chunk:
					break
					
				o.write(chunk)
				
			for s in sections:
				if s.cryptoType == 1: #plain text
					continue
					
				if s.cryptoType != 3:
					raise IOError('unknown crypto type')
					
				print('%x - %d bytes, type %d' % (s.offset, s.size, s.cryptoType))
				
				i = s.offset
				
				crypto = AESCTR(s.cryptoKey, s.cryptoCounter)
				end = s.offset + s.size
				
				while i < end:
					o.seek(i)
					crypto.seek(i)
					chunkSz = 0x10000 if end - i > 0x10000 else end - i
					buf = o.read(chunkSz)
					
					if not len(buf):
						break
					
					o.seek(i)
					o.write(crypto.encrypt(buf))
					
					i += chunkSz