Golang snappy.Decode函数代码示例

func Fuzz(data []byte) int {
	n, err := snappy.DecodedLen(data)
	if err != nil || n > 1e6 {
		return 0
	}
	if n < 0 {
		panic("negative decoded len")
	}
	dec, err := snappy.Decode(nil, data)
	if err != nil {
		if dec != nil {
			panic("dec is not nil")
		}
		return 0
	}
	if len(dec) != n {
		println(len(dec), n)
		panic("bad decoded len")
	}
	n = snappy.MaxEncodedLen(len(dec))
	enc := snappy.Encode(nil, dec)
	if len(enc) > n {
		panic("bad encoded len")
	}

	dec1, err := snappy.Decode(nil, enc)
	if err != nil {
		panic(err)
	}
	if bytes.Compare(dec, dec1) != 0 {
		panic("not equal")
	}
	return 1
}

func (*Segment) snappyDecode(src []byte) ([]byte, error) {
	if bytes.Equal(src[:8], snappyMagic) {
		var (
			pos   = uint32(16)
			max   = uint32(len(src))
			dst   = make([]byte, 0, len(src))
			chunk []byte
			err   error
		)
		for pos < max {
			size := binary.BigEndian.Uint32(src[pos : pos+4])
			pos += 4

			chunk, err = snappy.Decode(chunk, src[pos:pos+size])
			if err != nil {
				return nil, err
			}
			pos += size
			dst = append(dst, chunk...)
		}

		return dst, nil
	}

	return snappy.Decode(nil, src)
}

func snappyDecode(b []byte) ([]byte, error) {
	if !bytes.HasPrefix(b, snappyJavaMagic) {
		return snappy.Decode(nil, b)
	}

	// See https://github.com/xerial/snappy-java/blob/develop/src/main/java/org/xerial/snappy/SnappyInputStream.java
	version := binary.BigEndian.Uint32(b[8:12])
	if version != 1 {
		return nil, fmt.Errorf("cannot handle snappy-java codec version other than 1 (got %d)", version)
	}
	// b[12:16] is the "compatible version"; ignore for now
	var (
		decoded = make([]byte, 0, len(b))
		chunk   []byte
		err     error
	)
	for i := 16; i < len(b); {
		n := int(binary.BigEndian.Uint32(b[i : i+4]))
		i += 4
		chunk, err = snappy.Decode(chunk, b[i:i+n])
		if err != nil {
			return nil, err
		}
		i += n
		decoded = append(decoded, chunk...)
	}
	return decoded, nil
}

// Decode decodes snappy data whether it is traditional unframed
// or includes the xerial framing format.
func Decode(src []byte) ([]byte, error) {
	if !bytes.Equal(src[:8], xerialHeader) {
		return master.Decode(nil, src)
	}

	var (
		pos   = uint32(16)
		max   = uint32(len(src))
		dst   = make([]byte, 0, len(src))
		chunk []byte
		err   error
	)
	for pos < max {
		size := binary.BigEndian.Uint32(src[pos : pos+4])
		pos += 4

		chunk, err = master.Decode(chunk, src[pos:pos+size])
		if err != nil {
			return nil, err
		}
		pos += size
		dst = append(dst, chunk...)
	}
	return dst, nil
}

func (e *Engine) unsnappy() error {
	data, err := snappy.Decode(nil, e.stack.Pop())
	if err == nil {
		e.stack.Push(data)
	}
	return err
}

func loadDigest(w http.ResponseWriter, r *http.Request, lg loghttp.FuncBufUniv, fs fsi.FileSystem, fnDigest string, treeX *DirTree) {

	fnDigestSnappied := strings.Replace(fnDigest, ".json", ".json.snappy", -1)
	bts, err := fs.ReadFile(fnDigestSnappied)
	if err == nil {
		btsDec := []byte{}
		lg("encoded digest loaded, size %vkB", len(bts)/1024)
		btsDec, err := snappy.Decode(nil, bts)
		if err != nil {
			lg(err)
			return
		}
		lg("digest decoded from %vkB to %vkB", len(bts)/1024, len(btsDec)/1024)
		bts = btsDec
	} else {
		bts, err = fs.ReadFile(fnDigest)
		lg(err)
	}

	if err == nil {
		err = json.Unmarshal(bts, &treeX)
		lg(err)
	}

	lg("DirTree   %5.2vkB loaded for %v", len(bts)/1024, fnDigest)

}

func TestSnappyCompressor(t *testing.T) {
	c := SnappyCompressor{}
	if c.Name() != "snappy" {
		t.Fatalf("expected name to be 'snappy', got %v", c.Name())
	}

	str := "My Test String"
	//Test Encoding
	expected := snappy.Encode(nil, []byte(str))
	if res, err := c.Encode([]byte(str)); err != nil {
		t.Fatalf("failed to encode '%v' with error %v", str, err)
	} else if bytes.Compare(expected, res) != 0 {
		t.Fatal("failed to match the expected encoded value with the result encoded value.")
	}

	val, err := c.Encode([]byte(str))
	if err != nil {
		t.Fatalf("failed to encode '%v' with error '%v'", str, err)
	}

	//Test Decoding
	if expected, err := snappy.Decode(nil, val); err != nil {
		t.Fatalf("failed to decode '%v' with error %v", val, err)
	} else if res, err := c.Decode(val); err != nil {
		t.Fatalf("failed to decode '%v' with error %v", val, err)
	} else if bytes.Compare(expected, res) != 0 {
		t.Fatal("failed to match the expected decoded value with the result decoded value.")
	}
}

// Load gets data from a file.
func (d *Data) Load() error {
	if fileExists(d.File) {
		// open file
		fh, err := os.OpenFile(d.File, os.O_RDONLY, 0655)
		if err != nil {
			return err
		}
		defer fh.Close()

		// read content
		rh, err := ioutil.ReadAll(fh)
		if err != nil {
			return err
		}

		// decompress content
		dh, err := snappy.Decode(nil, rh)
		if err != nil {
			return err
		}

		// decode content
		gh := bytes.NewReader(dh)

		dedata := gob.NewDecoder(gh)
		err = dedata.Decode(&d.Store)

		if err != nil {
			return err
		}
	}

	return nil
}

func (b *EncryptBackend) Get(hash string) (data []byte, err error) {
	ref, _ := b.index[hash]
	enc, err := b.dest.Get(ref)
	if err != nil {
		return data, err
	}
	box := enc[headerSize:]
	var nonce [24]byte
	encData := make([]byte, len(box)-24)
	copy(nonce[:], box[:24])
	copy(encData[:], box[24:])
	out := make([]byte, len(box)-24)
	out, success := secretbox.Open(nil, encData, &nonce, b.key)
	if !success {
		return data, fmt.Errorf("failed to decrypt blob %v/%v", hash, ref)
	}
	// Decode snappy data
	data, err = snappy.Decode(nil, out)
	if err != nil {
		return data, fmt.Errorf("failed to decode blob %v/%v", hash, ref)
	}
	blobsDownloaded.Add(b.dest.String(), 1)
	bytesDownloaded.Add(b.dest.String(), int64(len(enc)))
	return
}

// Next indicates if there is a value to read
func (r *WALSegmentReader) Next() bool {
	b := getBuf(defaultBufLen)
	defer putBuf(b)
	var nReadOK int

	// read the type and the length of the entry
	n, err := io.ReadFull(r.r, b[:5])
	if err == io.EOF {
		return false
	}

	if err != nil {
		r.err = err
		// We return true here because we want the client code to call read which
		// will return the this error to be handled.
		return true
	}
	nReadOK += n

	entryType := b[0]
	length := btou32(b[1:5])

	// read the compressed block and decompress it
	if int(length) > len(b) {
		b = make([]byte, length)
	}

	n, err = io.ReadFull(r.r, b[:length])
	if err != nil {
		r.err = err
		return true
	}
	nReadOK += n

	data, err := snappy.Decode(nil, b[:length])
	if err != nil {
		r.err = err
		return true
	}

	// and marshal it and send it to the cache
	switch WalEntryType(entryType) {
	case WriteWALEntryType:
		r.entry = &WriteWALEntry{
			Values: map[string][]Value{},
		}
	case DeleteWALEntryType:
		r.entry = &DeleteWALEntry{}
	default:
		r.err = fmt.Errorf("unknown wal entry type: %v", entryType)
		return true
	}
	r.err = r.entry.UnmarshalBinary(data)
	if r.err == nil {
		// Read and decode of this entry was successful.
		r.n += int64(nReadOK)
	}

	return true
}

func (c *CompressionSnappyDecoder) readHeader() (int, error) {
	header := make([]byte, 4, 4)
	_, err := c.source.Read(header[:3])
	if err != nil {
		return 0, err
	}
	headerVal := binary.LittleEndian.Uint32(header)
	c.isOriginal = headerVal%2 == 1
	c.chunkLength = int(headerVal / 2)
	if !c.isOriginal {
		// ORC does not use snappy's framing as implemented in the
		// github.com/golang/snappy Reader implementation. As a result
		// we have to read and decompress the entire chunk.
		// TODO: find reader implementation with optional framing.
		r := io.LimitReader(c.source, int64(c.chunkLength))
		src, err := ioutil.ReadAll(r)
		if err != nil {
			return 0, err
		}
		decodedBytes, err := snappy.Decode(nil, src)
		if err != nil {
			return 0, err
		}
		c.decoded = bytes.NewReader(decodedBytes)
	} else {
		c.decoded = io.LimitReader(c.source, int64(c.chunkLength))
	}
	return 0, nil
}

func (db *Database) getDoc(c redis.Conn, path string) (*doc.Package, time.Time, error) {
	r, err := redis.Values(getDocScript.Do(c, path))
	if err == redis.ErrNil {
		return nil, time.Time{}, nil
	} else if err != nil {
		return nil, time.Time{}, err
	}

	var p []byte
	var t int64

	if _, err := redis.Scan(r, &p, &t); err != nil {
		return nil, time.Time{}, err
	}

	p, err = snappy.Decode(nil, p)
	if err != nil {
		return nil, time.Time{}, err
	}

	var pdoc doc.Package
	if err := gob.NewDecoder(bytes.NewReader(p)).Decode(&pdoc); err != nil {
		return nil, time.Time{}, err
	}

	nextCrawl := pdoc.Updated
	if t != 0 {
		nextCrawl = time.Unix(t, 0).UTC()
	}

	return &pdoc, nextCrawl, err
}

func SnappyMustUnCompress(inb []byte) (outb []byte) {
	outb, err := snappy.Decode(nil, inb)
	if err != nil {
		panic(err)
	}
	return outb
}

// Read next record. The returned slice may be a sub-slice of dst if dst was
// large enough to hold the entire record. Otherwise, a newly allocated slice
// will be returned. It's valid to pass nil dst
func (rr *Reader) ReadRecord(dst []byte) (output []byte, err error) {
	if rr.Err != nil {
		return nil, rr.Err
	}
	headerBytes := [recordHeaderStorageSize]byte{}
	if _, err = io.ReadFull(rr.bytesReader, headerBytes[:]); err != nil {
		return nil, rr.err(err)
	}
	header := recordHeader{}
	if err = header.decode(headerBytes[:]); err != nil {
		return nil, rr.err(err)
	}
	if header.flags&NoCompression != 0 {
		return rr.readBody(header, dst)
	}
	rawBuf, err := rr.readBody(header, rr.uncompressBuf)
	if err != nil {
		return nil, err
	}
	buf, err := snappy.Decode(dst, rawBuf)
	if err != nil {
		return nil, rr.err(ErrReadBytes)
	}
	return buf, nil
}

func readRequestBody(r io.Reader, header *wire.RequestHeader, request proto.Message) error {
	// recv body (end)
	compressedPbRequest, err := recvFrame(r)
	if err != nil {
		return err
	}

	// checksum
	if crc32.ChecksumIEEE(compressedPbRequest) != header.GetChecksum() {
		return fmt.Errorf("protorpc.readRequestBody: unexpected checksum.")
	}

	// decode the compressed data
	pbRequest, err := snappy.Decode(nil, compressedPbRequest)
	if err != nil {
		return err
	}
	// check wire header: rawMsgLen
	if uint32(len(pbRequest)) != header.GetRawRequestLen() {
		return fmt.Errorf("protorpc.readRequestBody: Unexcpeted header.RawRequestLen.")
	}

	// Unmarshal to proto message
	if request != nil {
		err = proto.Unmarshal(pbRequest, request)
		if err != nil {
			return err
		}
	}

	return nil
}