Python zstandard.ZstdCompressor方法代码示例

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def open_tar_zst(path, mode):
    if mode == "w":
        cctx = zstandard.ZstdCompressor(threads=-1)
        with open(path, "wb") as f:
            with cctx.stream_writer(f) as compressor:
                with tarfile.open(mode="w|", fileobj=compressor) as tar:
                    yield tar
    elif mode == "r":
        dctx = zstandard.ZstdDecompressor()
        with open(path, "rb") as f:
            with dctx.stream_reader(f) as reader:
                with tarfile.open(mode="r|", fileobj=reader) as tar:
                    yield tar
    else:
        assert False, f"Unexpected mode: {mode}"


# Using tar directly is twice as fast than through Python! 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def zstandard_file(infile, mode="rb"):
        if "r" in mode:
            cctx = zstd.ZstdDecompressor()
            return cctx.stream_reader(infile)
        else:
            cctx = zstd.ZstdCompressor(level=10)
            return cctx.stream_writer(infile) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def compress(data):
        # ZstdCompressor is not thread safe.
        # TODO: Use a pool?
        return ZstdCompressor().compress(data) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def zstd_compress(path):
    cctx = zstandard.ZstdCompressor(threads=-1)
    with open(path, "rb") as input_f:
        with open(f"{path}.zst", "wb") as output_f:
            cctx.copy_stream(input_f, output_f) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def _db_open(path, mode):
    parts = str(path).split(".")
    assert len(parts) > 1, "Extension needed to figure out serialization format"
    if len(parts) == 2:
        db_format = parts[-1]
        compression = None
    else:
        db_format = parts[-2]
        compression = parts[-1]

    assert compression is None or compression in COMPRESSION_FORMATS
    assert db_format in SERIALIZATION_FORMATS

    store_constructor = SERIALIZATION_FORMATS[db_format]

    if compression == "gz":
        with gzip.GzipFile(path, mode) as f:
            yield store_constructor(f)
    elif compression == "zstd":
        if "w" in mode or "a" in mode:
            cctx = zstandard.ZstdCompressor()
            with open(path, mode) as f:
                with cctx.stream_writer(f) as writer:
                    yield store_constructor(writer)
        else:
            dctx = zstandard.ZstdDecompressor()
            with open(path, mode) as f:
                with dctx.stream_reader(f) as reader:
                    yield store_constructor(reader)
    else:
        with open(path, mode) as f:
            yield store_constructor(f) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def mock_zst():
    def create_zst_file(db_path, content=b'{"Hello": "World"}'):
        with open(db_path, "wb") as output_f:
            cctx = zstandard.ZstdCompressor()
            with cctx.stream_writer(output_f) as compressor:
                compressor.write(content)

    return create_zst_file 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def __init__(self, src_fp, algorithm, level=0):
        super().__init__(src_fp, minimum_read_size=32 * 1024)
        if algorithm == "lzma":
            self._compressor = lzma.LZMACompressor(lzma.FORMAT_XZ, -1, level, None)
        elif algorithm == "snappy":
            self._compressor = snappy.StreamCompressor()
        elif algorithm == "zstd":
            self._compressor = zstd.ZstdCompressor(level=level).compressobj()
        else:
            InvalidConfigurationError("invalid compression algorithm: {!r}".format(algorithm)) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def __init__(self, next_fp, level, threads=0):
        self._zstd = zstd.ZstdCompressor(level=level, threads=threads).compressobj()
        super().__init__(next_fp) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def gcp(repository, revision, report, platform, suite):
    """
    Upload a grcov raw report on Google Cloud Storage
    * Compress with zstandard
    * Upload on bucket using revision in name
    * Trigger ingestion on channel's backend
    """
    assert isinstance(report, bytes)
    assert isinstance(platform, str)
    assert isinstance(suite, str)
    bucket = get_bucket(secrets[secrets.GOOGLE_CLOUD_STORAGE])

    # Compress report
    compressor = zstd.ZstdCompressor()
    archive = compressor.compress(report)

    # Upload archive
    path = GCP_COVDIR_PATH.format(
        repository=repository, revision=revision, platform=platform, suite=suite
    )
    blob = bucket.blob(path)
    blob.upload_from_string(archive)

    # Update headers
    blob.content_type = "application/json"
    blob.content_encoding = "zstd"
    blob.patch()

    logger.info("Uploaded {} on {}".format(path, bucket))

    # Trigger ingestion on backend
    gcp_ingest(repository, revision, platform, suite)

    return blob 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def mock_bucket(mock_secrets):
    """
    Mock a GCP bucket & blobs
    """

    class MockBlob(object):
        def __init__(self, name, content=None, exists=False):
            self.name = name
            if content is not None:
                assert isinstance(content, bytes)

                # Auto zstandard compression
                if self.name.endswith(".zstd"):
                    compressor = zstd.ZstdCompressor()
                    self._content = compressor.compress(content)
                else:
                    self._content = content
            else:
                self._content = None
            self._exists = exists

        def exists(self):
            return self._exists

        def download_to_filename(self, path):
            assert self._exists and self._content
            with open(path, "wb") as f:
                f.write(self._content)

    class MockBucket(object):
        _blobs = {}

        def add_mock_blob(self, name, coverage=0.0):
            content = json.dumps({"coveragePercent": coverage, "children": {}}).encode(
                "utf-8"
            )
            self._blobs[name] = MockBlob(name, content, exists=True)

        def blob(self, name):
            if name in self._blobs:
                return self._blobs[name]
            return MockBlob(name)

    return MockBucket() 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def itkimage_to_json(itkimage, manager=None):
    """Serialize a Python itk.Image object.

    Attributes of this dictionary are to be passed to the JavaScript itkimage
    constructor.
    """
    if itkimage is None:
        return None
    else:
        direction = itkimage.GetDirection()
        directionMatrix = direction.GetVnlMatrix()
        directionList = []
        dimension = itkimage.GetImageDimension()
        pixel_arr = itk.array_view_from_image(itkimage)
        componentType, pixelType = _image_to_type(itkimage)
        if 'int64' in componentType:
            # JavaScript does not yet support 64-bit integers well
            if componentType == 'uint64_t':
                pixel_arr = pixel_arr.astype(np.uint32)
                componentType = 'uint32_t'
            else:
                pixel_arr = pixel_arr.astype(np.int32)
                componentType = 'int32_t'
        compressor = zstd.ZstdCompressor(level=3)
        compressed = compressor.compress(pixel_arr.data)
        pixel_arr_compressed = memoryview(compressed)
        for col in range(dimension):
            for row in range(dimension):
                directionList.append(directionMatrix.get(row, col))
        imageType = dict(
            dimension=dimension,
            componentType=componentType,
            pixelType=pixelType,
            components=itkimage.GetNumberOfComponentsPerPixel()
        )
        return dict(
            imageType=imageType,
            origin=tuple(itkimage.GetOrigin()),
            spacing=tuple(itkimage.GetSpacing()),
            size=tuple(itkimage.GetBufferedRegion().GetSize()),
            direction={'data': directionList,
                       'rows': dimension,
                       'columns': dimension},
            compressedData=pixel_arr_compressed
        ) 

# 需要导入模块: import zstandard [as 别名]
# 或者: from zstandard import ZstdCompressor [as 别名]
def polydata_list_to_json(polydata_list, manager=None):  # noqa: C901
    """Serialize a list of a Python object that represents vtk.js PolyData.

    The returned data is compatibile with vtk.js PolyData with compressed data
    buffers.
    """
    if polydata_list is None:
        return None
    else:
        compressor = zstd.ZstdCompressor(level=3)

        json = []
        for polydata in polydata_list:
            json_polydata = dict()
            for top_key, top_value in polydata.items():
                if isinstance(top_value, dict):
                    nested_value_copy = dict()
                    for nested_key, nested_value in top_value.items():
                        if not nested_key == 'values':
                            nested_value_copy[nested_key] = nested_value
                    json_polydata[top_key] = nested_value_copy
                else:
                    json_polydata[top_key] = top_value

            if 'points' in json_polydata:
                point_values = polydata['points']['values']
                compressed = compressor.compress(point_values.data)
                compressedView = memoryview(compressed)
                json_polydata['points']['compressedValues'] = compressedView

            for cell_type in ['verts', 'lines', 'polys', 'strips']:
                if cell_type in json_polydata:
                    values = polydata[cell_type]['values']
                    compressed = compressor.compress(values.data)
                    compressedView = memoryview(compressed)
                    json_polydata[cell_type]['compressedValues'] = compressedView

            for data_type in ['pointData', 'cellData']:
                if data_type in json_polydata:
                    data = polydata[data_type]
                    compressed_data = dict()
                    for nested_key, nested_value in data.items():
                        if not nested_key == 'arrays':
                            compressed_data[nested_key] = nested_value
                    compressed_arrays = []
                    for array in polydata[data_type]['arrays']:
                        compressed_array = dict()
                        for nested_key, nested_value in array['data'].items():
                            if not nested_key == 'values':
                                compressed_array[nested_key] = nested_value
                        values = array['data']['values']
                        compressed = compressor.compress(values.data)
                        compressedView = memoryview(compressed)
                        compressed_array['compressedValues'] = compressedView
                        compressed_arrays.append({'data': compressed_array})
                    compressed_data['arrays'] = compressed_arrays
                    json_polydata[data_type] = compressed_data

            json.append(json_polydata)
        return json