Golang snappy.Decode函数代码示例

// readAll read entire referenced block.
func (p *bInfo) readAll(r io.ReaderAt, checksum bool) (b []byte, err error) {
	raw := make([]byte, p.size+5)
	_, err = readFullAt(r, raw, int64(p.offset))
	if err != nil {
		return
	}

	crcb := raw[len(raw)-4:]
	raw = raw[:len(raw)-4]

	if checksum {
		sum := binary.LittleEndian.Uint32(crcb)
		sum = hash.UnmaskCRC32(sum)
		crc := hash.NewCRC32C()
		crc.Write(raw)
		if crc.Sum32() != sum {
			err = errors.ErrCorrupt("block checksum mismatch")
			return
		}
	}

	compression := raw[len(raw)-1]
	b = raw[:len(raw)-1]

	switch compression {
	case kNoCompression:
	case kSnappyCompression:
		return snappy.Decode(nil, b)
	default:
		err = errors.ErrCorrupt("bad block type")
	}

	return
}

func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) {
	data := r.bpool.Get(int(bh.length + blockTrailerLen))
	if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF {
		return nil, err
	}
	if checksum || r.checksum {
		if !verifyChecksum(data) {
			return nil, errors.New("leveldb/table: Reader: invalid block (checksum mismatch)")
		}
	}
	switch data[bh.length] {
	case blockTypeNoCompression:
		data = data[:bh.length]
	case blockTypeSnappyCompression:
		decLen, err := snappy.DecodedLen(data[:bh.length])
		if err != nil {
			return nil, err
		}
		tmp := data
		data, err = snappy.Decode(r.bpool.Get(decLen), tmp[:bh.length])
		r.bpool.Put(tmp)
		if err != nil {
			return nil, err
		}
	default:
		return nil, fmt.Errorf("leveldb/table: Reader: unknown block compression type: %d", data[bh.length])
	}
	return data, nil
}

func readRequestBody(r io.Reader, header *wire.RequestHeader, request proto.Message) error {
	// recv body (end)
	compressedPbRequest, err := recvFrame(r)
	if err != nil {
		return err
	}

	// checksum
	if crc32.ChecksumIEEE(compressedPbRequest) != header.GetChecksum() {
		return fmt.Errorf("protorpc.readRequestBody: unexpected checksum.")
	}

	// decode the compressed data
	pbRequest, err := snappy.Decode(nil, compressedPbRequest)
	if err != nil {
		return err
	}
	// check wire header: rawMsgLen
	if uint32(len(pbRequest)) != header.GetRawRequestLen() {
		return fmt.Errorf("protorpc.readRequestBody: Unexcpeted header.RawRequestLen.")
	}

	// Unmarshal to proto message
	if request != nil {
		err = proto.Unmarshal(pbRequest, request)
		if err != nil {
			return err
		}
	}

	return nil
}

func (ev *CompressEvent) Decode(buffer *bytes.Buffer) (err error) {
	ev.CompressType, err = DecodeUInt32Value(buffer)
	if err != nil {
		return
	}
	switch ev.CompressType {
	case COMPRESSOR_NONE:
		err, ev.Ev = DecodeEvent(buffer)
		return err
	case COMPRESSOR_SNAPPY:
		b := make([]byte, 0, 0)
		b, err = snappy.Decode(b, buffer.Bytes())
		if err != nil {
			b = nil
			return
		}
		tmpbuf := bytes.NewBuffer(b)
		err, ev.Ev = DecodeEvent(tmpbuf)
		tmpbuf.Reset()
		return err
		//	case COMPRESSOR_LZ4:
		//		lz4r := lz4.NewReader(buffer)
		//		data, _ := ioutil.ReadAll(lz4r)
		//		tmpbuf := bytes.NewBuffer(data)
		//		err, ev.Ev = DecodeEvent(tmpbuf)
		//		tmpbuf.Reset()
		//		return err
	default:
		return errors.New("Not supported compress type:" + strconv.Itoa(int(ev.CompressType)))
	}
	return nil
}

func Decompress(src []byte) ([]byte, bool) {
	dst, err := snappy.Decode(nil, src)
	if err != nil {
		return nil, false
	}
	return dst, true
}

// Do executes function f for each document in the database.
func (db *Database) Do(f func(*doc.Package, []Package) error) error {
	c := db.Pool.Get()
	defer c.Close()
	keys, err := redis.Values(c.Do("KEYS", "pkg:*"))
	if err != nil {
		return err
	}
	for _, key := range keys {
		p, err := redis.Bytes(c.Do("HGET", key, "gob"))
		if err == redis.ErrNil {
			continue
		}
		if err != nil {
			return err
		}
		p, err = snappy.Decode(nil, p)
		if err != nil {
			return err
		}
		var pdoc doc.Package
		if err := gob.NewDecoder(bytes.NewReader(p)).Decode(&pdoc); err != nil {
			return err
		}
		pkgs, err := db.getSubdirs(c, pdoc.ImportPath, &pdoc)
		if err != nil {
			return err
		}
		if err := f(&pdoc, pkgs); err != nil {
			return err
		}
	}
	return nil
}

func (db *Database) getDoc(c redis.Conn, path string) (*doc.Package, time.Time, error) {
	r, err := redis.Values(getDocScript.Do(c, path))
	if err == redis.ErrNil {
		return nil, time.Time{}, nil
	} else if err != nil {
		return nil, time.Time{}, err
	}

	var p []byte
	var t int64

	if _, err := redis.Scan(r, &p, &t); err != nil {
		return nil, time.Time{}, err
	}

	p, err = snappy.Decode(nil, p)
	if err != nil {
		return nil, time.Time{}, err
	}

	var pdoc doc.Package
	if err := gob.NewDecoder(bytes.NewReader(p)).Decode(&pdoc); err != nil {
		return nil, time.Time{}, err
	}

	nextCrawl := pdoc.Updated
	if t != 0 {
		nextCrawl = time.Unix(t, 0).UTC()
	}

	return &pdoc, nextCrawl, err
}

func (this *SnappyCodec) Inverse(src, dst []byte) (uint, uint, error) {
	if src == nil {
		return uint(0), uint(0), errors.New("Invalid null source buffer")
	}

	if dst == nil {
		return uint(0), uint(0), errors.New("Invalid null destination buffer")
	}

	if kanzi.SameByteSlices(src, dst, false) {
		return 0, 0, errors.New("Input and output buffers cannot be equal")
	}

	count := this.size

	if this.size == 0 {
		count = uint(len(src))
	}

	res, err := snappy.Decode(dst, src[0:count])

	if err != nil {
		return 0, 0, fmt.Errorf("Decoding error: %v", err)
	}

	if len(res) > len(dst) {
		// Encode returns a newly allocated slice if the provided 'dst' array is too small.
		// There is no way to return this new slice, so treat it as an error
		return 0, 0, fmt.Errorf("Output buffer is too small - size: %d, required %d", len(res), len(dst))
	}

	return count, uint(len(res)), nil
}

func TestSnappyCompressor(t *testing.T) {
	c := SnappyCompressor{}
	if c.Name() != "snappy" {
		t.Fatalf("expected name to be 'snappy', got %v", c.Name())
	}

	str := "My Test String"
	//Test Encoding
	if expected, err := snappy.Encode(nil, []byte(str)); err != nil {
		t.Fatalf("failed to encode '%v' with error %v", str, err)
	} else if res, err := c.Encode([]byte(str)); err != nil {
		t.Fatalf("failed to encode '%v' with error %v", str, err)
	} else if bytes.Compare(expected, res) != 0 {
		t.Fatal("failed to match the expected encoded value with the result encoded value.")
	}

	val, err := c.Encode([]byte(str))
	if err != nil {
		t.Fatalf("failed to encode '%v' with error '%v'", str, err)
	}

	//Test Decoding
	if expected, err := snappy.Decode(nil, val); err != nil {
		t.Fatalf("failed to decode '%v' with error %v", val, err)
	} else if res, err := c.Decode(val); err != nil {
		t.Fatalf("failed to decode '%v' with error %v", val, err)
	} else if bytes.Compare(expected, res) != 0 {
		t.Fatal("failed to match the expected decoded value with the result decoded value.")
	}
}

func SnappyUncompress(compressed []byte) []byte {
	dst := make([]byte, 0, len(compressed))
	out, err := snappy.Decode(dst, compressed)
	if err != nil {
		panic(err)
	}
	return out
}

func (t tweetItem) Write(out io.Writer) {
	v := t.Value()
	t, err := snappy.Decode(nil, v)
	if err != nil {
		panic(err)
	}
	out.Write(t)
	out.Write([]byte("\n"))
}

func (cn *connection) recv() (byte, []byte, error) {
	if cn.c == nil {
		return 0, nil, driver.ErrBadConn
	}
	header := make([]byte, 8)
	if _, err := io.ReadFull(cn.c, header); err != nil {
		cn.close() // better assume that the connection is broken (may have read some bytes)
		return 0, nil, err
	}
	// verify that the frame starts with version==1 and req/resp flag==response
	// this may be overly conservative in that future versions may be backwards compatible
	// in that case simply amend the check...
	if header[0] != protoResponse {
		cn.close()
		return 0, nil, fmt.Errorf("unsupported frame version or not a response: 0x%x (header=%v)", header[0], header)
	}
	// verify that the flags field has only a single flag set, again, this may
	// be overly conservative if additional flags are backwards-compatible
	if header[1] > 1 {
		cn.close()
		return 0, nil, fmt.Errorf("unsupported frame flags: 0x%x (header=%v)", header[1], header)
	}
	opcode := header[3]
	if opcode > opLAST {
		cn.close()
		return 0, nil, fmt.Errorf("unknown opcode: 0x%x (header=%v)", opcode, header)
	}
	length := binary.BigEndian.Uint32(header[4:8])
	var body []byte
	if length > 0 {
		if length > 256*1024*1024 { // spec says 256MB is max
			cn.close()
			return 0, nil, fmt.Errorf("frame too large: %d (header=%v)", length, header)
		}
		body = make([]byte, length)
		if _, err := io.ReadFull(cn.c, body); err != nil {
			cn.close() // better assume that the connection is broken
			return 0, nil, err
		}
	}
	if header[1]&flagCompressed != 0 && cn.compression == "snappy" {
		var err error
		body, err = snappy.Decode(nil, body)
		if err != nil {
			cn.close()
			return 0, nil, err
		}
	}
	if opcode == opError {
		code := binary.BigEndian.Uint32(body[0:4])
		msglen := binary.BigEndian.Uint16(body[4:6])
		msg := string(body[6 : 6+msglen])
		return opcode, body, Error{Code: int(code), Msg: msg}
	}
	return opcode, body, nil
}

func (r *Reader) loadBlock() error {
	ref := r.refs[0]
	r.refs = r.refs[1:]

	path := blockPath(ref)
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	n, err := f.Read(r.box)
	if err != nil && err != io.EOF {
		return err
	}
	//if err != io.EOF {
	//	//TODO instead we can read the actual block stored disregarding its size.
	//	return fmt.Errorf("didn't read full block: BlockSize is smaller than stored?")
	//}
	if n < minBoxSize {
		return fmt.Errorf("block on disk is too short: %s", ref)
	}
	// Decrypt.
	var nonce [24]byte
	if err := readNonce(&nonce, r.box); err != nil {
		return err
	}
	encryptedBlock := r.box[len(nonce):n]
	decryptedData, ok := secretbox.Open(r.cdata[:0], encryptedBlock, &nonce, &config.Keys.BlockEnc)
	if !ok {
		return fmt.Errorf("failed to decrypt block %s", ref)
	}

	// Load block kind.
	r.kind = decryptedData[0]
	// Load length of compressed data.
	compressedLen := binary.BigEndian.Uint32(decryptedData[1:])

	decryptedData = decryptedData[headerSize : headerSize+compressedLen]

	// Decompress.
	// TODO avoid allocation.
	decompressedData, err := snappy.Decode(nil, decryptedData)
	if err != nil {
		return err
	}

	// Verify hash.
	contentHash := calculateRef(r.h, decompressedData)
	if !ref.Equal(contentHash) {
		return fmt.Errorf("block ref %s doesn't match content %s", ref, contentHash)
	}

	// Set block.
	r.block = decompressedData
	return nil
}

// SnappyDecode decodes snappy data
func SnappyDecode(src []byte) ([]byte, error) {
	if bytes.Equal(src[:8], snappyMagic) {
		pos := uint32(16)
		max := uint32(len(src))
		dst := make([]byte, 0)
		for pos < max {
			size := binary.BigEndian.Uint32(src[pos : pos+4])
			pos = pos + 4
			chunk, err := snappy.Decode(nil, src[pos:pos+size])
			if err != nil {
				return nil, err
			}
			pos = pos + size
			dst = append(dst, chunk...)
		}
		return dst, nil
	}
	return snappy.Decode(nil, src)
}

func KVQuery(db *rpcplus.Client, table, uid string, value interface{}) (exist bool, err error) {
	//ts("KVQuery", table, uid)
	//defer te("KVQuery", table, uid)

	var reply protobuf.DBQueryResult

	err = db.Call("DBServer.Query", protobuf.DBQuery{table, uid}, &reply)

	if err != nil {
		logger.Error("KVQuery Error On Query %s : %s (%s)", table, uid, err.Error())
		return
	}

	switch reply.Code {
	case protobuf.Ok:

		var dst []byte

		dst, err = snappy.Decode(nil, reply.Value)

		if err != nil {
			logger.Error("KVQuery Unmarshal Error On snappy.Decode %s : %s (%s)", table, uid, err.Error())
			return
		}

		switch value.(type) {
		case gp.Message:

			err = gp.Unmarshal(dst, value.(gp.Message))

			if err != nil {
				logger.Error("KVQuery Unmarshal Error On Query %s : %s (%s)", table, uid, err.Error())
				return
			}
		case *[]byte:

			*(value.(*[]byte)) = dst

		default:
			logger.Error("KVQuery args type error %v", value)
			return
		}

		exist = true
		return

	case protobuf.NoExist:
		return
	}

	logger.Error("KVQuery Unknow DBReturn %d", reply.Code)

	return false, fmt.Errorf("KVQuery Unknow DBReturn %d", reply.Code)
}