Python blosc.compress方法代码示例

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def unconvert(values, dtype, compress=None):

    if dtype == np.object_:
        return np.array(values, dtype=object)

    if compress == 'zlib':

        values = zlib.decompress(values)
        return np.frombuffer(values, dtype=dtype)

    elif compress == 'blosc':

        if not _BLOSC:
            raise Exception("cannot uncompress w/o blosc")

        # decompress
        values = blosc.decompress(values)

        return np.frombuffer(values, dtype=dtype)

    # from a string
    return np.fromstring(values.encode('latin1'), dtype=dtype) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def render(self, data, media_type=None, renderer_context=None):

        if renderer_context['response'].status_code == 403:
            renderer_context["accepted_media_type"] = 'application/json'
            self.media_type = 'application/json'
            self.format = 'json'
            err_msg = {"status": 403, "message": "Access denied, are you logged in?",
                       "code": 2005}
            jr = JSONRenderer()
            return jr.render(err_msg, 'application/json', renderer_context)

        if not data["data"].data.flags['C_CONTIGUOUS']:
            data["data"].data = np.ascontiguousarray(data["data"].data, dtype=data["data"].data.dtype)

        # Return data, squeezing time dimension if only a single point
        if data["time_request"]:
            return blosc.compress(data["data"].data, typesize=renderer_context['view'].bit_depth)
        else:
            return blosc.compress(np.squeeze(data["data"].data, axis=(0,)),
                                  typesize=renderer_context['view'].bit_depth) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def test_channel_uint8_wrong_data_type(self):
        """ Test posting the wrong bitdepth data """

        config = bossutils.configuration.BossConfig()

        test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint16)
        h = test_mat.tobytes()
        bb = blosc.compress(h, typesize=16)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb,
                               content_type='application/blosc')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1',
                                    resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def test_channel_uint8_wrong_dimensions(self):
        """ Test posting with the wrong xyz dims"""

        test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint8)
        h = test_mat.tobytes()
        bb = blosc.compress(h, typesize=8)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:100/0:128/0:16/', bb,
                               content_type='application/blosc')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1',
                                    resolution='0', x_range='0:100', y_range='0:128', z_range='0:16', t_range=None)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def test_channel_uint16_wrong_data_type(self):
        """ Test posting the wrong bitdepth data """

        test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint8)
        h = test_mat.tobytes()
        bb = blosc.compress(h, typesize=8)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb,
                               content_type='application/blosc')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2',
                                    resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def test_channel_uint16_wrong_dimensions(self):
        """ Test posting with the wrong xyz dims"""

        test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint16)
        h = test_mat.tobytes()
        bb = blosc.compress(h, typesize=16)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/0:100/0:128/0:16/', bb,
                               content_type='application/blosc')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2',
                                    resolution='0', x_range='0:100', y_range='0:128', z_range='0:16', t_range=None)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def test_channel_uint64_wrong_data_type(self):
        """ Test posting the wrong bitdepth data """

        test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint8)
        h = test_mat.tobytes()
        bb = blosc.compress(h, typesize=8)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:16/', bb,
                               content_type='application/blosc')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
                                    resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def test_channel_uint64_wrong_dimensions(self):
        """ Test posting with the wrong xyz dims"""

        test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint64)
        h = test_mat.tobytes()
        bb = blosc.compress(h, typesize=64)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:100/0:128/0:16/', bb,
                               content_type='application/blosc')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
                                    resolution='0', x_range='0:100', y_range='0:128', z_range='0:16', t_range=None)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def commit_ref_db_val_from_raw_val(db_kvs: Iterable[Tuple[bytes, bytes]]) -> DigestAndBytes:
    """serialize and compress a list of db_key/db_value pairs for commit storage

    Parameters
    ----------
    db_kvs : Iterable[Tuple[bytes, bytes]]
        Iterable collection binary encoded db_key/db_val pairs.

    Returns
    -------
    DigestAndBytes
        `raw` serialized and compressed representation of the object. `digest`
        digest of the joined db kvs.
    """
    joined = tuple(map(CMT_KV_JOIN_KEY.join, db_kvs))
    refDigest = _commit_ref_joined_kv_digest(joined)
    pck = CMT_REC_JOIN_KEY.join(joined)
    raw = blosc.compress(pck, typesize=1, clevel=8, shuffle=blosc.NOSHUFFLE, cname='zstd')
    return DigestAndBytes(digest=refDigest, raw=raw) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def to_bytes(self, compress: bool=True) -> bytes:
        """Method to serialize to bytes for storage in the activity detail db

        Returns:
            bytes
        """
        dict_data = self.to_dict(compact=True)

        # Serialize data items
        serializer_obj = Serializer()
        for mime_type in dict_data['d']:
            dict_data['d'][mime_type] = serializer_obj.serialize(mime_type, dict_data['d'][mime_type])

            # Compress object data
            if compress:
                if type(dict_data['d'][mime_type]) != bytes:
                    raise ValueError("Data must be serialized to bytes before compression")

                dict_data['d'][mime_type] = blosc.compress(dict_data['d'][mime_type], typesize=8,
                                                           cname='blosclz',
                                                           shuffle=blosc.SHUFFLE)

        # Base64 encode binary data while dumping to json string
        return json.dumps(dict_data, cls=ActivityDetailRecordEncoder, separators=(',', ':')).encode('utf-8') 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def convert(values):
    """ convert the numpy values to a list """

    dtype = values.dtype

    if is_categorical_dtype(values):
        return values

    elif is_object_dtype(dtype):
        return values.ravel().tolist()

    if needs_i8_conversion(dtype):
        values = values.view('i8')
    v = values.ravel()

    if compressor == 'zlib':
        _check_zlib()

        # return string arrays like they are
        if dtype == np.object_:
            return v.tolist()

        # convert to a bytes array
        v = v.tostring()
        return ExtType(0, zlib.compress(v))

    elif compressor == 'blosc':
        _check_blosc()

        # return string arrays like they are
        if dtype == np.object_:
            return v.tolist()

        # convert to a bytes array
        v = v.tostring()
        return ExtType(0, blosc.compress(v, typesize=dtype.itemsize))

    # ndarray (on original dtype)
    return ExtType(0, v.tostring()) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def _save_file(f, data, compressor='zstd'):
    assert isinstance(data, np.ndarray), "Please pass a numpy array"
    d_comp = COMPRESSORS[compressor]['compress'](data)
    f.write(d_comp)
    return len(d_comp) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def to_msgpack(path_or_buf, *args, **kwargs):
    """
    msgpack (serialize) object to input file path

    THIS IS AN EXPERIMENTAL LIBRARY and the storage format
    may not be stable until a future release.

    Parameters
    ----------
    path_or_buf : string File path, buffer-like, or None
                  if None, return generated string
    args : an object or objects to serialize
    append : boolean whether to append to an existing msgpack
             (default is False)
    compress : type of compressor (zlib or blosc), default to None (no
               compression)
    """
    global compressor
    compressor = kwargs.pop('compress', None)
    append = kwargs.pop('append', None)
    if append:
        mode = 'a+b'
    else:
        mode = 'wb'

    def writer(fh):
        for a in args:
            fh.write(pack(a, **kwargs))

    if isinstance(path_or_buf, compat.string_types):
        with open(path_or_buf, mode) as fh:
            writer(fh)
    elif path_or_buf is None:
        buf = compat.BytesIO()
        writer(buf)
        return buf.getvalue()
    else:
        writer(path_or_buf) 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def convert(values):
    """ convert the numpy values to a list """

    dtype = values.dtype
    if needs_i8_conversion(dtype):
        values = values.view('i8')
    v = values.ravel()

    # convert object
    if dtype == np.object_:
        return v.tolist()

    if compressor == 'zlib':

        # return string arrays like they are
        if dtype == np.object_:
            return v.tolist()

        # convert to a bytes array
        v = v.tostring()
        return zlib.compress(v)

    elif compressor == 'blosc' and _BLOSC:

        # return string arrays like they are
        if dtype == np.object_:
            return v.tolist()

        # convert to a bytes array
        v = v.tostring()
        return blosc.compress(v, typesize=dtype.itemsize)

    # ndarray (on original dtype)
    return v.tostring() 

# 需要导入模块: import blosc [as 别名]
# 或者: from blosc import compress [as 别名]
def _dump_blosc(self, ix, dst, components=None):
        """ Save blosc packed data to file """
        file_name = self._get_file_name(ix, dst)
        with open(file_name, 'w+b') as f:
            if self.components is None:
                components = (None,)
                item = (self[ix],)
            else:
                components = tuple(components or self.components)
                item = self[ix].as_tuple(components)
            data = dict(zip(components, item))
            f.write(blosc.compress(dill.dumps(data)))