Golang snappy.NewReader函数代码示例

func TestPut(t *testing.T) {
	s := NewStore("test", &nullStreamReader{}, nil)

	testData := []byte{0x01, 0x02, 0x03}

	err := s.Put(testData)
	if err != nil {
		t.Errorf("Failed to put %v", err)
	}

	fname := *s.currentFilename
	defer os.Remove(fname)

	s.Close()

	f, err := os.Open(fname)
	if err != nil {
		t.Errorf("Failed to open")
		return
	}

	df := snappy.NewReader(f)
	data, err := ioutil.ReadAll(df)
	if err != nil {
		t.Errorf("Failed to read %v", err)
	} else {
		if bytes.Compare(data, testData) != 0 {
			t.Errorf("Data mismatch")
		}
	}
}

func TestBuildWriteValueRequest(t *testing.T) {
	assert := assert.New(t)
	input1, input2 := "abc", "def"
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte(input1)),
		chunks.NewChunk([]byte(input2)),
	}

	hints := map[hash.Hash]struct{}{
		hash.Parse("sha1-0000000000000000000000000000000000000002"): struct{}{},
		hash.Parse("sha1-0000000000000000000000000000000000000003"): struct{}{},
	}
	compressed := buildWriteValueRequest(serializeChunks(chnx, assert), hints)
	gr := snappy.NewReader(compressed)

	count := 0
	for hint := range deserializeHints(gr) {
		count++
		_, present := hints[hint]
		assert.True(present)
	}
	assert.Equal(len(hints), count)

	chunkChan := make(chan *chunks.Chunk, 16)
	go chunks.DeserializeToChan(gr, chunkChan)
	for c := range chunkChan {
		assert.Equal(chnx[0].Hash(), c.Hash())
		chnx = chnx[1:]
	}
	assert.Empty(chnx)
}

func newCompStream(conn net.Conn) *compStream {
	c := new(compStream)
	c.conn = conn
	c.w = snappy.NewBufferedWriter(conn)
	c.r = snappy.NewReader(conn)
	return c
}

func main() {
	http.HandleFunc("/receive", func(w http.ResponseWriter, r *http.Request) {
		reqBuf, err := ioutil.ReadAll(snappy.NewReader(r.Body))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		var req remote.WriteRequest
		if err := proto.Unmarshal(reqBuf, &req); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		for _, ts := range req.Timeseries {
			m := make(model.Metric, len(ts.Labels))
			for _, l := range ts.Labels {
				m[model.LabelName(l.Name)] = model.LabelValue(l.Value)
			}
			fmt.Println(m)

			for _, s := range ts.Samples {
				fmt.Printf("  %f %d\n", s.Value, s.TimestampMs)
			}
		}
	})

	http.ListenAndServe(":1234", nil)
}

func (suite *LevelDBPutCacheSuite) extractChunks(hashes hashSet) <-chan *chunks.Chunk {
	buf := &bytes.Buffer{}
	err := suite.cache.ExtractChunks(hashes, buf)
	suite.NoError(err)

	chunkChan := make(chan *chunks.Chunk)
	go chunks.DeserializeToChan(snappy.NewReader(buf), chunkChan)
	return chunkChan
}

func runRemote() {
	if *destinationFlag == "" {
		log.Println("Must specify destination in receive mode")
		os.Exit(1)
	}
	if *timestampFlag == "" {
		log.Println("Must specify timestamp in receive mode")
		os.Exit(1)
	}
	var snapshotsLoc SnapshotsLoc
	snapshotsLoc.Directory = *destinationFlag
	snapshotsLoc.Limits = Limits{
		Hourly:  *hourlyFlag,
		Daily:   *dailyFlag,
		Weekly:  *weeklyFlag,
		Monthly: *monthlyFlag}

	lock, err := NewDirLock(snapshotsLoc.Directory)
	if err != nil {
		log.Println(err.Error())
		return
	}
	defer lock.Unlock()
	timestamp := Timestamp(*timestampFlag)
	_, err = parseTimestamp(timestamp)
	if err != nil {
		log.Println(err.Error())
		os.Exit(1)
	}
	if verbosity > 2 {
		log.Println("runRemote: ReceiveAndCleanUp")
	}
	var runner CmdRunner
	if *noCompressionFlag {
		runner = snapshotsLoc.ReceiveAndCleanUp(os.Stdin, timestamp)
	} else {
		rd := snappy.NewReader(os.Stdin)
		runner = snapshotsLoc.ReceiveAndCleanUp(rd, timestamp)
	}
	err = <-runner.Started
	if verbosity > 2 {
		log.Println("runRemote: ReceiveAndCleanUp Started")
	}
	if err != nil {
		log.Println(err.Error())
		os.Exit(1)
	}
	err = <-runner.Done
	if verbosity > 2 {
		log.Println("runRemote: ReceiveAndCleanUp Done")
	}
	if err != nil {
		log.Println(err.Error())
		os.Exit(1)
	}
}

func do(isDecompress bool, filename, suffix string, isToStdout bool) (percentage, speed float64, err error) {
	var (
		input   io.Reader
		output  io.Writer
		outName string = "-"
	)
	if filename == "-" {
		input = os.Stdin
		output = os.Stdout
	} else {
		fi, err := os.Open(filename)
		if err != nil {
			return 0, 0, err
		}
		input = fi
		defer fi.Close()

		if isToStdout {
			output = os.Stdout
		} else {
			if isDecompress {
				if !strings.HasSuffix(filename, suffix) {
					err = errors.New(fmt.Sprintf("file: %s not has suffix %s", filename, suffix))
					return 0, 0, err
				}
				outName = filename[:(len(filename) - len(suffix))]
			} else {
				outName = filename + suffix
			}
			fo, err := os.Create(outName)
			if err != nil {
				return 0, 0, err
			}
			output = fo
			defer fo.Close()
		}
	}
	start := time.Now()
	rwc := NewRWCounter(input, output)
	if isDecompress {
		_, err = io.Copy(rwc, snappy.NewReader(rwc))
	} else {
		_, err = io.Copy(snappy.NewWriter(rwc), rwc)
	}
	useTime := time.Since(start).Seconds()
	if isDecompress {
		percentage = 1 - float64(rwc.CountR())/float64(rwc.CountW())
		speed = float64(rwc.CountW()) / 1024.0 / 1024.0 / useTime
	} else {
		percentage = 1 - float64(rwc.CountW())/float64(rwc.CountR())
		speed = float64(rwc.CountR()) / 1024.0 / 1024.0 / useTime
	}
	return
}

func resBodyReader(res *http.Response) (reader io.ReadCloser) {
	reader = res.Body
	if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
		gr, err := gzip.NewReader(reader)
		d.Chk.NoError(err)
		reader = gr
	} else if strings.Contains(res.Header.Get("Content-Encoding"), "x-snappy-framed") {
		sr := snappy.NewReader(reader)
		reader = ioutil.NopCloser(sr)
	}
	return
}

func bodyReader(req *http.Request) (reader io.ReadCloser) {
	reader = req.Body
	if strings.Contains(req.Header.Get("Content-Encoding"), "gzip") {
		gr, err := gzip.NewReader(reader)
		d.PanicIfError(err)
		reader = gr
	} else if strings.Contains(req.Header.Get("Content-Encoding"), "x-snappy-framed") {
		sr := snappy.NewReader(reader)
		reader = ioutil.NopCloser(sr)
	}
	return
}

func (f *File) decode(bs []byte, target interface{}) (err error) {
	var r io.Reader
	if f.compressMethod == _COMPRESS_SNAPPY {
		r = snappy.NewReader(bytes.NewReader(bs))
	} else {
		r = bytes.NewReader(bs)
	}
	if f.codec == _CODEC_GOB {
		return gob.NewDecoder(r).Decode(target)
	} else if f.codec == _CODEC_MSGPACK {
		return msgpack.NewDecoder(r).Decode(target)
	}
	return fmt.Errorf("not reachable")
}

// Get can be called from any goroutine to retrieve the chunk referenced by hash. If the chunk is not present, Get returns the empty Chunk.
func (p *orderedChunkCache) Get(hash hash.Hash) chunks.Chunk {
	// Don't use defer p.mu.RUnlock() here, because I want reading from orderedChunks NOT to be guarded by the lock. LevelDB handles its own goroutine-safety.
	p.mu.RLock()
	dbKey, ok := p.chunkIndex[hash]
	p.mu.RUnlock()

	if !ok {
		return chunks.EmptyChunk
	}
	data, err := p.orderedChunks.Get(dbKey, nil)
	d.Chk.NoError(err)
	reader := snappy.NewReader(bytes.NewReader(data))
	chunkChan := make(chan *chunks.Chunk)
	go chunks.DeserializeToChan(reader, chunkChan)
	return *(<-chunkChan)
}

func (c *Conn) upgradeSnappy() error {
	conn := net.Conn(c.conn)
	if c.tlsConn != nil {
		conn = c.tlsConn
	}
	c.r = snappy.NewReader(conn)
	c.w = snappy.NewWriter(conn)
	frameType, data, err := ReadUnpackedResponse(c)
	if err != nil {
		return err
	}
	if frameType != FrameTypeResponse || !bytes.Equal(data, []byte("OK")) {
		return errors.New("invalid response from Snappy upgrade")
	}
	return nil
}

func NewSnappyResponseReader(resp *http.Response) io.ReadCloser {
	var reader io.Reader
	reader = resp.Body
	if resp.Header.Get("Content-Encoding") == "snappy" {
		if isr := snappyReaderPool.Get(); isr != nil {
			sr := isr.(*snappy.Reader)
			sr.Reset(reader)
			reader = sr
		} else {
			// Creates a new one if the pool is empty
			reader = snappy.NewReader(reader)
		}
	}
	return &snappyResponseReader{
		resp:   resp,
		Reader: reader,
	}
}

// Decompress a snappy archive.
func unsnap(src *os.File) (dst *os.File, err error) {
	srcInfo, err := src.Stat()
	if err != nil {
		return
	}
	srcName := srcInfo.Name()

	// Make sure existing files are not overwritten.
	dstName := strings.TrimSuffix(srcName, ".sz")

	getUnusedFilename(&dstName)
	print(concat(srcName, "  >  ", dstName))

	// Create the destination file.
	dst, err = create(dstName, srcInfo.Mode())
	if err != nil {
		return
	}
	// Remember to re-open the uncompressed file after it has been written.
	defer func() {
		if err == nil {
			dst, err = os.Open(dstName)
		}
	}()

	pt := &passthru{
		Reader:    src,
		nExpected: uint64(srcInfo.Size()),
	}
	defer pt.Reset()

	szr := snappy.NewReader(pt)
	defer szr.Reset(nil)

	defer print()
	_, err = io.Copy(dst, szr)
	return
}

func NewArchiveReader(ir io.Reader) (or Reader) {
	sr := snappy.NewReader(ir)
	mr := msgp.NewReader(sr)

	return &ArchiveReader{mr}
}