Golang crc32.Update函数代码示例

func TestNeedle(t *testing.T) {
	var (
		err       error
		n, tn     *Needle
		br        *bufio.Reader
		data1     = []byte("tes1")
		checksum1 = crc32.Update(0, _crc32Table, data1)
		data2     = []byte("tes2")
		checksum2 = crc32.Update(0, _crc32Table, data2)
		buf       = &bytes.Buffer{}
	)
	// WriteFrom
	if _, err = buf.Write(data1); err != nil {
		t.Error(err)
		t.FailNow()
	}
	n = NewWriter(3, 3, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Error(err)
		t.FailNow()
	}
	t.Log(n)
	tn = new(Needle)
	tn.buffer = n.Buffer()
	// Parse
	if err = tn.Parse(); err != nil {
		t.Error(err)
		t.FailNow()
	}
	t.Log(n)
	compareNeedle(t, tn, 3, 3, data1, FlagOK, checksum1)
	buf.Write(data2)
	n = NewWriter(4, 4, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Error(err)
		t.FailNow()
	}
	tn = new(Needle)
	tn.buffer = n.Buffer()
	if err = tn.Parse(); err != nil {
		t.Error(err)
		t.FailNow()
	}
	compareNeedle(t, tn, 4, 4, data2, FlagOK, checksum2)
	// ParseFrom
	if _, err = buf.Write(n.Buffer()); err != nil {
		t.Error(err)
		t.FailNow()
	}
	br = bufio.NewReader(buf)
	tn = new(Needle)
	if err = tn.ParseFrom(br); err != nil {
		t.Error(err)
		t.FailNow()
	}
	t.Log(tn)
	compareNeedle(t, tn, 4, 4, data2, FlagOK, checksum2)
}

func (w *Writer) finishBlock() (blockHandle, error) {
	if w.nEntries == 0 {
		w.restarts = w.restarts[:1]
		w.restarts[0] = 0
	}
	tmp4 := w.tmp[:4]
	for _, x := range w.restarts {
		binary.LittleEndian.PutUint32(tmp4, x)
		w.buf.Write(tmp4)
	}
	binary.LittleEndian.PutUint32(tmp4, uint32(len(w.restarts)))
	w.buf.Write(tmp4)

	b := w.buf.Bytes()
	w.tmp[0] = 0

	checksum := crc32.Update(0, table, b)
	checksum = crc32.Update(checksum, table, w.tmp[:1])
	checksum = uint32(checksum>>15|checksum<<17) + 0xa282ead8
	binary.LittleEndian.PutUint32(w.tmp[1:5], checksum)

	if _, err := w.writer.Write(b); err != nil {
		return blockHandle{}, err
	}
	if _, err := w.writer.Write(w.tmp[:5]); err != nil {
		return blockHandle{}, err
	}
	bh := blockHandle{int64(w.offset), int64(len(b))}
	w.offset += uint64(len(b)) + BlockTrailerLen

	w.buf.Reset()
	w.nEntries = 0
	w.restarts = w.restarts[:0]
	return bh, nil
}

func calcNewCRC32(path string, newHeader *BinaryHeader) (uint32, error) {
	buf := &bytes.Buffer{}
	err := binary.Write(buf, binary.BigEndian, newHeader)
	if err != nil {
		return 0, err
	}

	header := buf.Bytes()
	crc := crc32.Checksum(header[:28], crc32.IEEETable) // before crc32
	crc = crc32.Update(crc, crc32.IEEETable, header[32:])

	f, err := os.Open(path)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	if _, err := f.Seek(int64(len(header)), os.SEEK_SET); err != nil {
		return 0, err
	}

	data := make([]byte, 4*1024*1024)
	for {
		n, err := f.Read(data)
		crc = crc32.Update(crc, crc32.IEEETable, data[:n])
		if err != nil {
			if err != io.EOF {
				return 0, err
			}
			break
		}
	}
	return crc, nil
}

// FillNeedleBuf fill needle buf with photo.
func FillNeedleBuf(key, cookie int64, data, buf []byte) (size int32) {
	var (
		n        int
		padding  int32
		checksum = crc32.Update(0, crc32Table, data)
	)
	size = int32(NeedleHeaderSize + len(data) + NeedleFooterSize)
	padding = NeedlePaddingSize - (size % NeedlePaddingSize)
	size += padding
	// header
	copy(buf[:needleMagicSize], needleHeaderMagic)
	n += needleMagicSize
	BigEndian.PutInt64(buf[n:], cookie)
	n += needleCookieSize
	BigEndian.PutInt64(buf[n:], key)
	n += needleKeySize
	buf[n] = NeedleStatusOK
	n += needleFlagSize
	BigEndian.PutInt32(buf[n:], int32(len(data)))
	n += needleSizeSize
	// data
	copy(buf[n:], data)
	n += len(data)
	// footer
	copy(buf[n:], needleFooterMagic)
	n += needleMagicSize
	BigEndian.PutUint32(buf[n:], checksum)
	n += needleChecksumSize
	copy(buf[n:], needlePadding[padding])
	return
}

// ParseNeedleData parse a needle data part.
func ParseNeedleData(buf []byte, n *Needle) (err error) {
	var (
		bn       int32
		checksum uint32
	)
	n.Data = buf[:n.Size]
	bn += n.Size
	n.FooterMagic = buf[bn : bn+needleMagicSize]
	if bytes.Compare(n.FooterMagic, needleFooterMagic) != 0 {
		err = ErrNeedleFooterMagic
		return
	}
	bn += needleMagicSize
	checksum = crc32.Update(0, crc32Table, n.Data)
	n.Checksum = BigEndian.Uint32(buf[bn : bn+needleChecksumSize])
	if n.Checksum != checksum {
		err = ErrNeedleChecksum
		return
	}
	bn += needleChecksumSize
	n.Padding = buf[bn : bn+n.PaddingSize]
	log.Printf("padding: %d, %v vs %v\n", n.PaddingSize, n.Padding, needlePadding[n.PaddingSize])
	if bytes.Compare(n.Padding, needlePadding[n.PaddingSize]) != 0 {
		err = ErrNeedlePaddingNotMatch
		return
	}
	return
}

func (r *Receiver) handleTransfer(conn io.ReadWriter) (
	transfer.TransferResults, error,
) {
	if _, err := conn.Write([]byte("ok")); err != nil {
		return transfer.TransferResults{}, err
	}

	res := transfer.TransferResults{}
	buffer := make([]byte, 1024)

	startTime := time.Now()
	for {
		n, err := conn.Read(buffer)
		if err != nil { // done reading
			break
		}

		res.BytesSent += uint32(n)
		res.Checksum = crc32.Update(res.Checksum, crc32.IEEETable, buffer)
	}
	endTime := time.Now()

	res.Duration = endTime.Sub(startTime)

	return res, nil
}

// WriteFrom Write needle from io.Reader into buffer.
func (ns *Needles) WriteFrom(key int64, cookie int32, size int32, rd io.Reader) (err error) {
	var (
		n            *Needle
		data         []byte
		headerOffset int32
		dataOffset   int32
		footerOffset int32
		endOffset    int32
	)
	if ns.wn >= ns.Num {
		return errors.ErrNeedleFull
	}
	n = &ns.needles[ns.wn]
	n.initSize(key, cookie, size)
	headerOffset = ns.ws
	dataOffset = headerOffset + _headerSize
	footerOffset = dataOffset + n.Size
	endOffset = footerOffset + n.FooterSize
	data = ns.buffer[dataOffset:footerOffset]
	// write into buffer header->data->footer
	if err = n.WriteHeader(ns.buffer[headerOffset:dataOffset]); err == nil {
		if _, err = rd.Read(data); err == nil {
			n.Data = data
			n.Checksum = crc32.Update(0, _crc32Table, data)
			err = n.WriteFooter(ns.buffer[footerOffset:endOffset])
		}
	}
	ns.wn++
	ns.ws += n.TotalSize
	return
}

// FillNeedle fill needle buffer.
func FillNeedle(padding, size int32, key, cookie int64, data, buf []byte) {
	var (
		n        int
		checksum = crc32.Update(0, crc32Table, data)
	)
	// --- header ---
	// magic
	copy(buf[:needleMagicSize], needleHeaderMagic)
	n += needleMagicSize
	// cookie
	BigEndian.PutInt64(buf[n:], cookie)
	n += needleCookieSize
	// key
	BigEndian.PutInt64(buf[n:], key)
	n += needleKeySize
	// flag
	buf[n] = NeedleStatusOK
	n += needleFlagSize
	// size
	BigEndian.PutInt32(buf[n:], size)
	n += needleSizeSize
	// data
	copy(buf[n:], data)
	n += len(data)
	// --- footer ---
	// magic
	copy(buf[n:], needleFooterMagic)
	n += needleMagicSize
	// checksum
	BigEndian.PutUint32(buf[n:], checksum)
	n += needleChecksumSize
	// padding
	copy(buf[n:], needlePadding[padding])
	return
}

func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
	start := time.Now()

	fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
	b := pbutil.MustMarshal(snapshot)
	crc := crc32.Update(0, crcTable, b)
	snap := snappb.Snapshot{Crc: crc, Data: b}
	d, err := snap.Marshal()
	if err != nil {
		return err
	} else {
		marshallingDurations.Observe(float64(time.Since(start)) / float64(time.Second))
	}

	err = pioutil.WriteAndSyncFile(path.Join(s.dir, fname), d, 0666)
	if err == nil {
		saveDurations.Observe(float64(time.Since(start)) / float64(time.Second))
	} else {
		err1 := os.Remove(path.Join(s.dir, fname))
		if err1 != nil {
			plog.Errorf("failed to remove broken snapshot file %s", path.Join(s.dir, fname))
		}
	}
	return err
}

// Read reads the snapshot named by snapname and returns the snapshot.
func Read(snapname string) (*raftpb.Snapshot, error) {
	b, err := ioutil.ReadFile(snapname)
	if err != nil {
		log.Printf("snap: snapshotter cannot read file %v: %v", snapname, err)
		return nil, err
	}

	var serializedSnap snappb.Snapshot
	if err = serializedSnap.Unmarshal(b); err != nil {
		log.Printf("snap: corrupted snapshot file %v: %v", snapname, err)
		return nil, err
	}

	if len(serializedSnap.Data) == 0 || serializedSnap.Crc == 0 {
		log.Printf("snap: unexpected empty snapshot")
		return nil, ErrEmptySnapshot
	}

	crc := crc32.Update(0, crcTable, serializedSnap.Data)
	if crc != serializedSnap.Crc {
		log.Printf("snap: corrupted snapshot file %v: crc mismatch", snapname)
		return nil, ErrCRCMismatch
	}

	var snap raftpb.Snapshot
	if err = snap.Unmarshal(serializedSnap.Data); err != nil {
		log.Printf("snap: corrupted snapshot file %v: %v", snapname, err)
		return nil, err
	}
	return &snap, nil
}

// Write writes a compressed form of p to the underlying io.Writer. The
// compressed bytes are not necessarily flushed until the Writer is closed.
func (z *Writer) Write(p []byte) (int, error) {
	if z.err != nil {
		return 0, z.err
	}
	var n int
	// Write the GZIP header lazily.
	if !z.wroteHeader {
		z.wroteHeader = true
		z.buf = [10]byte{0: gzipID1, 1: gzipID2, 2: gzipDeflate}
		if z.Extra != nil {
			z.buf[3] |= 0x04
		}
		if z.Name != "" {
			z.buf[3] |= 0x08
		}
		if z.Comment != "" {
			z.buf[3] |= 0x10
		}
		if z.ModTime.After(time.Unix(0, 0)) {
			// Section 2.3.1, the zero value for MTIME means that the
			// modified time is not set.
			le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix()))
		}
		if z.level == BestCompression {
			z.buf[8] = 2
		} else if z.level == BestSpeed {
			z.buf[8] = 4
		}
		z.buf[9] = z.OS
		n, z.err = z.w.Write(z.buf[:10])
		if z.err != nil {
			return n, z.err
		}
		if z.Extra != nil {
			z.err = z.writeBytes(z.Extra)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.Name != "" {
			z.err = z.writeString(z.Name)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.Comment != "" {
			z.err = z.writeString(z.Comment)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.compressor == nil {
			z.compressor, _ = flate.NewWriter(z.w, z.level)
		}
	}
	z.size += uint32(len(p))
	z.digest = crc32.Update(z.digest, crc32.IEEETable, p)
	n, z.err = z.compressor.Write(p)
	return n, z.err
}

// ParseNeedleData parse a needle data part.
func (n *Needle) ParseData(buf []byte) (err error) {
	var (
		bn       int32
		checksum uint32
	)
	n.Data = buf[:n.Size]
	bn += n.Size
	n.FooterMagic = buf[bn : bn+magicSize]
	if !bytes.Equal(n.FooterMagic, footerMagic) {
		err = errors.ErrNeedleFooterMagic
		return
	}
	bn += magicSize
	checksum = crc32.Update(0, crc32Table, n.Data)
	n.Checksum = binary.BigEndian.Uint32(buf[bn : bn+checksumSize])
	if n.Checksum != checksum {
		err = errors.ErrNeedleChecksum
		return
	}
	bn += checksumSize
	n.Padding = buf[bn : bn+n.PaddingSize]
	if !bytes.Equal(n.Padding, padding[n.PaddingSize]) {
		err = errors.ErrNeedlePadding
	}
	return
}

// readString reads a NUL-terminated string from z.r.
// It treats the bytes read as being encoded as ISO 8859-1 (Latin-1) and
// will output a string encoded using UTF-8.
// This method always updates z.digest with the data read.
func (z *Reader) readString() (string, error) {
	var err error
	needConv := false
	for i := 0; ; i++ {
		if i >= len(z.buf) {
			return "", ErrHeader
		}
		z.buf[i], err = z.r.ReadByte()
		if err != nil {
			return "", err
		}
		if z.buf[i] > 0x7f {
			needConv = true
		}
		if z.buf[i] == 0 {
			// Digest covers the NUL terminator.
			z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:i+1])

			// Strings are ISO 8859-1, Latin-1 (RFC 1952, section 2.3.1).
			if needConv {
				s := make([]rune, 0, i)
				for _, v := range z.buf[:i] {
					s = append(s, rune(v))
				}
				return string(s), nil
			}
			return string(z.buf[:i]), nil
		}
	}
}

// PushRow is part of the RowReceiver interface.
func (hr *hashRouter) PushRow(row sqlbase.EncDatumRow) bool {
	if hr.err != nil {
		return false
	}
	hr.buffer = hr.buffer[:0]
	for _, col := range hr.hashCols {
		if int(col) >= len(row) {
			hr.err = errors.Errorf("hash column %d, stream with only %d columns", col, len(row))
			return false
		}
		// TODO(radu): we should choose an encoding that is already available as
		// much as possible. However, we cannot decide this locally as multiple
		// nodes may be doing the same hashing and the encodings need to match. The
		// encoding needs to be determined at planning time.
		hr.buffer, hr.err = row[col].Encode(&hr.alloc, preferredEncoding, hr.buffer)
		if hr.err != nil {
			return false
		}
	}

	// We use CRC32-C because it makes for a decent hash function and is faster
	// than most hashing algorithms (on recent x86 platforms where it is hardware
	// accelerated).
	streamIdx := crc32.Update(0, crc32Table, hr.buffer) % uint32(len(hr.streams))

	// We can't return false if this stream happened to not need any more rows. We
	// could only return false once all streams returned false, but that seems of
	// limited benefit.
	_ = hr.streams[streamIdx].PushRow(row)
	return true
}

func TestNeedles(t *testing.T) {
	var (
		err       error
		tn        *Needle
		data1     = []byte("tes1")
		checksum1 = crc32.Update(0, _crc32Table, data1)
		data2     = []byte("tes2")
		checksum2 = crc32.Update(0, _crc32Table, data2)
		ns        = NewNeedles(2)
		buf       = &bytes.Buffer{}
	)
	if _, err = buf.Write(data1); err != nil {
		t.FailNow()
	}
	if err = ns.ReadFrom(1, 1, 4, buf); err != nil {
		t.FailNow()
	}
	if _, err = buf.Write(data2); err != nil {
		t.FailNow()
	}
	if err = ns.ReadFrom(2, 2, 4, buf); err != nil {
		t.FailNow()
	}
	tn = new(Needle)
	tn.buffer = ns.Next().Buffer()
	if err = tn.Parse(); err != nil {
		t.FailNow()
	}
	t.Log(tn)
	compareNeedle(t, tn, 1, 1, data1, FlagOK, checksum1)
	tn = new(Needle)
	tn.buffer = ns.Next().Buffer()
	if err = tn.Parse(); err != nil {
		t.FailNow()
	}
	t.Log(tn)
	compareNeedle(t, tn, 2, 2, data2, FlagOK, checksum2)
	if err = ns.ReadFrom(3, 3, 4, buf); err != errors.ErrNeedleFull {
		t.FailNow()
	}
	if tn = ns.Next(); tn != nil {
		t.FailNow()
	}
}