Golang go-lz4.Encode函数代码示例

func main() {

	verbose := flag.Bool("v", false, "verbose")

	flag.Parse()

	scanner := bufio.NewScanner(os.Stdin)

	w := bufio.NewWriter(os.Stdout)
	defer w.Flush()

	for scanner.Scan() {

		fname := scanner.Text()

		b, err := ioutil.ReadFile(fname)
		if err != nil {
			log.Println(err)
			continue
		}

		sip := siphash.Hash(0, 0, b)

		data, err := lz4.Encode(nil, b)
		if err != nil {
			log.Println("error packing: ", err)
			continue
		}

		const headerSize = 4 + 8 + 2 + 4 // + len(fname)

		h := struct {
			PacketSize       uint32
			SipHash          uint64
			Fname            []byte `binpack:"lenprefix=uint16"`
			UncompressedSize uint32
		}{
			PacketSize:       uint32(headerSize + len(fname) + len(data)),
			SipHash:          sip,
			Fname:            []byte(fname),
			UncompressedSize: uint32(len(b)),
		}

		if *verbose {
			log.Printf("%s: %d -> %d\n", fname, len(b), len(data))
		}

		err = binpack.Write(w, binary.LittleEndian, h)
		if err != nil {
			log.Printf("error writing header %+v\n", err)
			break
		}

		_, err = w.Write(data)
		if err != nil {
			log.Printf("error writing %+v\n", err)
			break
		}
	}
}

func (w *lz4Writer) Write(bs []byte) (int, error) {
	w.mut.Lock()
	defer w.mut.Unlock()

	var err error
	w.buf, err = lz4.Encode(w.buf[:cap(w.buf)], bs)
	if err != nil {
		return 0, err
	}

	var hdr [8]byte
	binary.BigEndian.PutUint32(hdr[0:], lz4Magic)
	binary.BigEndian.PutUint32(hdr[4:], uint32(len(w.buf)))
	_, err = w.wr.Write(hdr[:])
	if err != nil {
		return 0, err
	}

	_, err = w.wr.Write(w.buf)
	if err != nil {
		return 0, err
	}

	if debug {
		l.Debugf("lz4 write; %d / %d bytes", len(bs), 8+len(w.buf))
	}
	return len(bs), nil
}

func (c *rawConnection) lz4Compress(src []byte) ([]byte, error) {
	var err error
	buf := buffers.get(len(src))
	buf, err = lz4.Encode(buf, src)
	if err != nil {
		return nil, err
	}

	binary.BigEndian.PutUint32(buf, binary.LittleEndian.Uint32(buf))
	return buf, nil
}

func main() {

	var optCPUProfile = flag.String("cpuprofile", "", "profile")
	flag.Parse()

	if *optCPUProfile != "" {
		f, err := os.Create(*optCPUProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	args := flag.Args()

	var data []byte

	if len(args) < 2 {
		fmt.Print("Usage: lz4 [-d] <input> <output>\n")
		os.Exit(1)
	}

	input, err := os.OpenFile(args[0], os.O_RDONLY, 0644)
	if err != nil {
		fmt.Printf("Failed to open input file %s\n", args[0])
		os.Exit(1)
	}
	defer input.Close()

	if *decompress {
		data, _ = ioutil.ReadAll(input)
		data, err = lz4.Decode(nil, data)
		if err != nil {
			fmt.Println("Failed to decode:", err)
			return
		}
	} else {
		data, _ = ioutil.ReadAll(input)
		data, err = lz4.Encode(nil, data)
		if err != nil {
			fmt.Println("Failed to encode:", err)
			return
		}
	}

	err = ioutil.WriteFile(args[1], data, 0644)
	if err != nil {
		fmt.Printf("Failed to open output file %s\n", args[1])
		os.Exit(1)
	}
}

func (c *rawConnection) writerLoop() {
	var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused
	var uncBuf []byte            // buffer for uncompressed message, kept and reused
	for {
		var tempBuf []byte
		var err error

		select {
		case hm := <-c.outbox:
			if hm.msg != nil {
				// Uncompressed message in uncBuf
				uncBuf, err = hm.msg.AppendXDR(uncBuf[:0])
				if err != nil {
					c.close(err)
					return
				}

				compress := false
				switch c.compression {
				case CompressAlways:
					compress = true
				case CompressMetadata:
					compress = hm.hdr.msgType != messageTypeResponse
				}

				if compress && len(uncBuf) >= compressionThreshold {
					// Use compression for large messages
					hm.hdr.compression = true

					// Make sure we have enough space for the compressed message plus header in msgBug
					msgBuf = msgBuf[:cap(msgBuf)]
					if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) {
						msgBuf = make([]byte, maxLen)
					}

					// Compressed is written to msgBuf, we keep tb for the length only
					tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf)
					binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf)))
					msgBuf = msgBuf[0 : len(tempBuf)+8]

					if debug {
						l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf))
					}
				} else {
					// No point in compressing very short messages
					hm.hdr.compression = false

					msgBuf = msgBuf[:cap(msgBuf)]
					if l := len(uncBuf) + 8; l > len(msgBuf) {
						msgBuf = make([]byte, l)
					}

					binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf)))
					msgBuf = msgBuf[0 : len(uncBuf)+8]
					copy(msgBuf[8:], uncBuf)

					if debug {
						l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf))
					}
				}
			} else {
				if debug {
					l.Debugf("write empty message; %v", hm.hdr)
				}
				binary.BigEndian.PutUint32(msgBuf[4:8], 0)
				msgBuf = msgBuf[:8]
			}

			binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr))

			if err == nil {
				var n int
				n, err = c.cw.Write(msgBuf)
				if debug {
					l.Debugf("wrote %d bytes on the wire", n)
				}
			}
			if err != nil {
				c.close(err)
				return
			}
		case <-c.closed:
			return
		}
	}
}