Python snappy.decompress函数代码示例

def snappy_decode(payload):
    if not has_snappy():
        raise NotImplementedError("Snappy codec is not available")

    if payload.startswith(_XERIAL_HEADER):
        # TODO ? Should become a fileobj ?
        view = memoryview(payload)
        out = []
        length = len(payload)

        cursor = 16
        while cursor < length:
            block_size = struct.unpack_from('!i', view, cursor)[0]
            # Skip the block size
            cursor += 4
            end = cursor + block_size
            # XXX snappy requires a bytes-like object but doesn't accept
            # a memoryview, so we must copy.
            out.append(snappy.decompress(view[cursor:end].tobytes()))
            cursor = end

        # See https://atleastfornow.net/blog/not-all-bytes/
        return b''.join(out)
    else:
        return snappy.decompress(payload)

def decode_snappy(buff):
    """Decode a buffer using Snappy

    If xerial is found to be in use, the buffer is decoded in a fashion
    compatible with the xerial snappy library.

    Adapted from kafka-python
    https://github.com/mumrah/kafka-python/pull/127/files
    """
    if snappy is None:
        raise ImportError("Please install python-snappy")
    if _detect_xerial_stream(buff):
        out = StringIO()
        body = buffer(buff[16:])
        length = len(body)
        cursor = 0
        while cursor < length:
            block_size = struct.unpack_from('!i', body[cursor:])[0]
            cursor += 4
            end = cursor + block_size
            out.write(snappy.decompress(body[cursor:end]))
            cursor = end
        out.seek(0)
        return out.read()
    else:
        return snappy.decompress(buff)

def snappy_unpack_blob(blob, sep=SEP):
    if len(blob) == 0: return None
    if blob[0] == 'S':
        return np.array(snappy.decompress(blob[1:]).split(sep))
    dt = lookup[blob[0]]
    arr = np.frombuffer(snappy.decompress(blob[1:]), dtype=dt)
    # hack since arrays arent writable from buffer and we need this for comp_het
    # phasing.
    if blob[0] == '?':
        arr.setflags(write=True)
    return arr

    def _get_data(cls, stream, encoding='utf-8', compress_option=None):
        if isinstance(stream, six.text_type):
            data = stream.encode(encoding)
        elif isinstance(stream, six.binary_type):
            data = stream
        else:
            data = stream.read()  # due to the restriction of protobuf api, just read the data all
            stream.close()  # directly close the stream
            if isinstance(data, six.text_type):
                data = data.encode(encoding)

        if compress_option is None or \
                compress_option.algorithm == CompressOption.CompressAlgorithm.ODPS_RAW:
            return data
        elif compress_option.algorithm == CompressOption.CompressAlgorithm.ODPS_ZLIB:
            return data  # because requests do the unzip automatically, thanks to them O.O
        elif compress_option.algorithm == CompressOption.CompressAlgorithm.ODPS_SNAPPY:
            try:
                import snappy
            except ImportError:
                raise errors.DependencyNotInstalledError(
                    'python-snappy library is required for snappy support')
            data = snappy.decompress(data)
            return data
        else:
            raise IOError('invalid compression option.')

    def _ReadChunkFromBevy(self, chunk_id, bevy, bevy_index, index_size):
        chunk_id_in_bevy = chunk_id % self.chunks_per_segment

        if index_size == 0:
            LOGGER.error("Index empty in %s: %s", self.urn, chunk_id)
            raise IOError("Index empty in %s: %s" % (self.urn, chunk_id))
        # The segment is not completely full.
        if chunk_id_in_bevy >= index_size:
            LOGGER.error("Bevy index too short in %s: %s",
                         self.urn, chunk_id)
            raise IOError("Bevy index too short in %s: %s" % (
                self.urn, chunk_id))

        # For the last chunk in the bevy, consume to the end of the bevy
        # segment.
        if chunk_id_in_bevy == index_size - 1:
            compressed_chunk_size = bevy.Size() - bevy.Tell()
        else:
            compressed_chunk_size = (bevy_index[chunk_id_in_bevy + 1] -
                                     bevy_index[chunk_id_in_bevy])

        bevy.Seek(bevy_index[chunk_id_in_bevy], 0)
        cbuffer = bevy.Read(compressed_chunk_size)
        if self.compression == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB:
            return zlib.decompress(cbuffer)

        if snappy and self.compression == lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY:
            return snappy.decompress(cbuffer)

        if self.compression == lexicon.AFF4_IMAGE_COMPRESSION_STORED:
            return cbuffer

        raise RuntimeError(
            "Unable to process compression %s" % self.compression)

def u_slug(username, slug):
    user = current_user
    post = user.posts.filter_by(slug=slug).first()
    if post:
        _prev = user.posts.filter(Post.created_timestamp < post.created_timestamp).slice(0, 4)
        _next = Post.query.filter(User.username==user.username,Post.created_timestamp > post.created_timestamp).order_by(Post.created_timestamp).slice(0, 4)
        _prev_count = _prev.count()
        _next_count = _next.count()

        if _prev_count < 2:
            _next = _next.slice(0, 4 - _prev_count)
        elif _next_count < 2:
            _prev = _prev.slice(0, 4 - _next_count)
        else:
            _prev = _prev.slice(0, 2)
            _next = _next.slice(0, 2)

        if post.content:
            # Decrypt
            half_key = session[generate_hash(user.user_key_salt)]
            key = xor_keys(half_key, app.config['MASTER_KEY'])
            content = AES_decrypt(key, post.content)
            content = snappy.decompress(content)
            return render_template("post.html", content=content, user=user, post=post, next=_next, prev=_prev)
        return render_template("post.html", content='', user=user, post=post, next=_next, prev=_prev)
    abort(404)

def UnpackState(packed_state):
  """Convert a packed State binary string into a StateStruct object. If the
  input doesn't have the STATE_MARK_ZIP prefix, it is assumed to be an old-style
  compressed state object, and is directly decompressed.

  Args:
    packed_state - Binary string of the type produces by PackState.

  Returns:
    Populated StateStruct object.
  """
  if not packed_state:
    return None

  if ord(packed_state[0]) == STATE_MARK_ZIP:
    # Extract the meta-data Struct from the packed data.
    meta = StateMeta()
    meta.Deserialize(packed_state)

    # Extract the compressed State from the packed data.
    compressed_state = packed_state[meta.Size():]

    # Compute the checksum and make sure it matches the metadata.
    cksum = zlib.adler32(compressed_state)
    if cksum != meta.checksum:
      raise ValueError('Compressed State Checksum Error')

    # Return the decompressed State.
    return pickle.loads(zlib.decompress(compressed_state))

  elif ord(packed_state[0]) == STATE_MARK_SNAPPY:
    # Extract the meta-data Struct from the packed data.
    meta = StateMeta()
    meta.Deserialize(packed_state)

    # Extract the compressed State from the packed data.
    compressed_state = packed_state[meta.Size():]

    # Compute the checksum and make sure it matches the metadata.
    cksum = zlib.adler32(compressed_state)
    if cksum != meta.checksum:
      raise ValueError('Compressed State Checksum Error')

    # Return the decompressed State.
    return pickle.loads(snappy.decompress(compressed_state))

  elif ord(packed_state[0]) == STATE_MARK_LIGHT:
    # Extract the meta-data Struct from the packed data.
    meta = StateMeta()
    meta.Deserialize(packed_state)

    # Extract the State buffer from the packed data.
    state_buffer = packed_state[meta.Size():]

    # Return the decompressed State.
    return pickle.load(state_buffer)

  else:
    # Unsupported format.
    raise ValueError('Unrecognized State serialization format')

def _read_page(fo, page_header, column_metadata):
    """Internal function to read the data page from the given file-object
    and convert it to raw, uncompressed bytes (if necessary)."""
    bytes_from_file = fo.read(page_header.compressed_page_size)
    codec = column_metadata.codec
    if codec is not None and codec != CompressionCodec.UNCOMPRESSED:
        if column_metadata.codec == CompressionCodec.SNAPPY:
            raw_bytes = snappy.decompress(bytes_from_file)
        elif column_metadata.codec == CompressionCodec.GZIP:
            io_obj = StringIO.StringIO(bytes_from_file)
            with gzip.GzipFile(fileobj=io_obj, mode='rb') as f:
                raw_bytes = f.read()
        else:
            raise ParquetFormatException(
                "Unsupported Codec: {0}".format(codec))
    else:
        raw_bytes = bytes_from_file
    logger.debug(
        "Read page with compression type {0}. Bytes {1} -> {2}".format(
        _get_name(CompressionCodec, codec),
        page_header.compressed_page_size,
        page_header.uncompressed_page_size))
    assert len(raw_bytes) == page_header.uncompressed_page_size, \
        "found {0} raw bytes (expected {1})".format(
            len(raw_bytes),
            page_header.uncompressed_page_size)
    return raw_bytes

    def get_subvolume(self, box_zyx, scale=0):
        """
        Fetch a subvolume from the remote BrainMaps volume.
        
        Args:
            box: (start, stop) tuple, in ZYX order.
            scale: Which scale to fetch the subvolume from.
        
        Returns:
            volume (ndarray), where volume.shape = (stop - start)
        """
        box_zyx = np.asarray(box_zyx)
        corner_zyx = box_zyx[0]
        shape_zyx = box_zyx[1] - box_zyx[0]
        
        corner_xyz = corner_zyx[::-1]
        shape_xyz = shape_zyx[::-1]
        
        snappy_data = fetch_subvol_data( self.http,
                                         self.project,
                                         self.dataset,
                                         self.volume_id,
                                         corner_xyz,
                                         shape_xyz,
                                         scale,
                                         self.change_stack_id,
                                         self.use_gzip )

        volume_buffer = snappy.decompress(snappy_data)
        volume = np.frombuffer(volume_buffer, dtype=self.dtype).reshape(shape_zyx)
        return volume

def decompress(x):
    # Luckily \x78\x9c is an invalid preamble for Snappy:
    # If the block was 120 bytes, the preamble would be \x78\x00.
    # The first byte cannot be \x78 in any other case.
    if x[0] == '\x78' and x[1] in ('\x9c', '\xda', '\x01'):
        return zlib.decompress(x)
    else:
        return snappy.decompress(x)

 def decode_ins_ops(self, event):
     """Parses the data field of a MicroEventLog event and returns
     a sequence of instruction ops (micro ops, grouped by instruction)."""
     assert event.HasField('micro_event_log')
     io_class = self._io_for_arch()
     fp = StringIO.StringIO(snappy.decompress(event.micro_event_log.data))
     with contextlib.closing(fp):
         for ins_op in InstructionOpsDecoder(io_class(fp)).decode_stream():
             yield ins_op

 def decode_micro_ops(self, event):
     """Parses the data field of a MicroEventLog event and returns
     a sequence of micro ops. """
     assert event.HasField('micro_event_log')
     io_class = self._io_for_arch()
     fp = StringIO.StringIO(snappy.decompress(event.micro_event_log.data))
     with contextlib.closing(fp):
         for op in MicroOpDecoder(io_class(fp)).decode_stream():
             yield op

 def untransform(self, buf):
     for trans_id in self.__read_transforms:
         if trans_id == self.ZLIB_TRANSFORM:
             buf = zlib.decompress(buf)
         elif trans_id == self.SNAPPY_TRANSFORM:
             buf = snappy.decompress(buf)
         if not trans_id in self.__write_transforms:
             self.__write_transforms.append(trans_id)
     return buf

def loads(classifier_ser):
    d = pickle.loads(snappy.decompress(classifier_ser))
    if d['classifier_name'] == 'plslinearsvmxval':
        def decision_function(x):
            for step_name, step in d['classifier'].steps[:-1]:
                x = step.transform(x)
            return d['classifier'].steps[-1][1].decision_function(x)
        d['classifier'].decision_function = decision_function
    return d['classifier']

def _get(key, callback, args):
    r = cache_table.find_one({'_id': key})
    if not r:
        content = callback(*args)
        data = bson.binary.Binary(snappy.compress(content))
        cache_table.insert_one({'_id': key, 'data': data})
    else:
        data = r['data']
    content = snappy.decompress(data)
    return content