Golang crc32.Checksum函数代码示例

// readBlock reads a full or partial block into p from rs. len(p) must be smaller than (bs - crc32Len).
// The checksum of the block is calculated and verified.
// ErrBadCRC is returned if the checksum is invalid.
func readBlock(rs io.ReadSeeker, p []byte, index, bs int64) (int64, error) {
	payloadLen := int(bs - crc32Len)
	if len(p) > payloadLen {
		return 0, ErrPayloadSizeTooLarge
	}
	b := make([]byte, crc32Len)

	_, err := rs.Seek(index*bs, os.SEEK_SET)
	if err != nil {
		return 0, err
	}

	n, err := rs.Read(b)
	// Cannot read full crc
	if n > 0 && n < 4 {
		return 0, ErrBadCRC
	}
	if err != nil {
		return 0, err
	}

	crc := binary.BigEndian.Uint32(b)
	if len(p) == payloadLen {
		// fast path for reading into a buffer of payload size
		n, err = rs.Read(p)
		if err != nil {
			return int64(n), err
		}
		// Invalid crc
		if crc != crc32.Checksum(p[:n], crc32cTable) {
			return 0, ErrBadCRC
		}
		return int64(n), nil
	} else {
		// p is smaller than payload size
		// read into another buffer (so we can compare full block CRC) and copy into p
		buf := make([]byte, payloadLen)
		n, err = rs.Read(buf)
		// If there is an eof returned, two cases may happen
		// 1. n < len(p) -- we don't have enough data to
		// fill into p, we should return the eof
		// 2. n >= len(p) -- we have enough data to fill
		// into p, shoudln't return the eof
		if err == io.EOF && n >= len(p) {
			err = nil
		}
		if err != nil {
			return int64(copy(p, buf[:n])), err
		}
		if crc != crc32.Checksum(buf[:n], crc32cTable) {
			return 0, ErrBadCRC
		}
		return int64(copy(p, buf[:n])), nil
	}
}

func (p fakePeers) PickPeer(key string) (peer ProtoGetter, ok bool) {
	if len(p) == 0 {
		return
	}
	n := crc32.Checksum([]byte(key), crc32.IEEETable) % uint32(len(p))
	return p[n], p[n] != nil
}

//一个packet包括包头和包体，保证在接收到包头后两秒钟内接收到包体，否则线程会一直阻塞
//因此，引入了超时机制
func RecvPackage(conn net.Conn) (packet *GetPacket, e error, isClose bool) {
	var data []byte
	header := make([]byte, 12)
	if conn == nil {
		// e = errors.New("")
		fmt.Println("连接已经关闭")
		isClose = true
		return
	}
	n, err := io.ReadFull(conn, header)
	if n == 0 && err == io.EOF {
		// fmt.Println("客户端断开连接")
		isClose = true
		e = err
		return
	} else if err != nil {
		// fmt.Println("接收数据出错:", err)
		isClose = true
		e = err
		return
	}
	//数据包长度
	size := binary.BigEndian.Uint32(header)
	//crc值
	crc1 := binary.BigEndian.Uint32(header[4:8])
	msgID := binary.BigEndian.Uint32(header[8:12])

	data = make([]byte, size)

	timeout := NewTimeOut(func() {

		n, err = io.ReadFull(conn, data)
		if uint32(n) != size {
			log.Println("数据包长度不正确", n, "!=", size)
			e = errors.New(fmt.Sprint("数据包长度不正确:%d!=%d", n, size))
			return
		}
		if err != nil {
			log.Println("读取数据出错:", err)
			e = err
			return
		}
		crc2 := crc32.Checksum(data, crc32.IEEETable)
		if crc1 != crc2 {
			log.Println("crc 数据验证不正确: ", crc1, " != ", crc2)
			e = errors.New(fmt.Sprint("crc 数据验证不正确:%d!=%d", crc1, crc2))
			return
		}
		packet = new(GetPacket)
		packet.Date = data
		packet.MsgID = int32(msgID)
		packet.Size = uint32(len(data))
	})
	isTimeOut := timeout.Do(time.Second * 5)
	if isTimeOut {
		e = errors.New("数据包头和数据包体不完整")
		return
	}
	return
}

// writeBlock writes a full or partial block into ws. len(p) must be smaller than (bs - crc32Len).
// Each block contains a crc32Len bytes crc32c of the payload and a (bs-crc32Len)
// bytes payload. Any error encountered is returned.
// It is caller's responsibility to ensure that only the last block in the file has
// len(p) < bs - crc32Len
func writeBlock(ws io.WriteSeeker, index, bs int64, p []byte) error {
	if int64(len(p)) > bs-crc32Len {
		return ErrPayloadSizeTooLarge
	}

	// seek to the beginning of the block
	_, err := ws.Seek(index*bs, os.SEEK_SET)
	if err != nil {
		return err
	}

	// write crc32c
	// TODO: reuse buffer
	b := make([]byte, crc32Len)
	binary.BigEndian.PutUint32(b, crc32.Checksum(p, crc32cTable))
	_, err = ws.Write(b)
	if err != nil {
		return err
	}

	// write payload
	_, err = ws.Write(p)
	if err != nil {
		return err
	}
	return nil
}

func get(key int64, circle map[uint32]string) string {
	keyStr := strconv.Itoa(int(key))
	if len(circle) == 0 {
		return ""
	} else {
		hash := crc32.Checksum([]byte(keyStr), crc32q)
		var keys []int
		for k := range circle {
			keys = append(keys, int(k))
		}
		sort.Ints(keys)
		var serverHash uint32
		foundHash := false
		for i := 0; i < len(keys); i++ {
			if keys[i] >= int(hash) {
				serverHash = uint32(keys[i])
				foundHash = true
				break
			}
		}
		if foundHash == true {
			return circle[serverHash]
		} else {
			return circle[uint32(keys[0])]
		}

	}
}

func (s *blockWriteStream) makePacket() outboundPacket {
	packetLength := outboundPacketSize
	if s.buf.Len() < outboundPacketSize {
		packetLength = s.buf.Len()
	}

	numChunks := int(math.Ceil(float64(packetLength) / float64(outboundChunkSize)))
	packet := outboundPacket{
		seqno:     s.seqno,
		offset:    s.offset,
		last:      false,
		checksums: make([]byte, numChunks*4),
		data:      make([]byte, packetLength),
	}

	// TODO: we shouldn't actually need this extra copy. We should also be able
	// to "reuse" packets.
	io.ReadFull(&s.buf, packet.data)

	// Fill in the checksum for each chunk of data.
	for i := 0; i < numChunks; i++ {
		chunkOff := i * outboundChunkSize
		chunkEnd := chunkOff + outboundChunkSize
		if chunkEnd >= len(packet.data) {
			chunkEnd = len(packet.data)
		}

		checksum := crc32.Checksum(packet.data[chunkOff:chunkEnd], crc32.IEEETable)
		binary.BigEndian.PutUint32(packet.checksums[i*4:], checksum)
	}

	return packet
}

func PacketData(msgID uint32, data *[]byte) *[]byte {
	buf := bytes.NewBuffer([]byte{})
	binary.Write(buf, binary.BigEndian, uint32(uint32(len(*data))))
	crc32 := crc32.Checksum(*data, crc32.IEEETable)
	binary.Write(buf, binary.BigEndian, crc32)
	// binary.Write(msgID, binary.BigEndian, )
	buf.Write(*data)
	bs := buf.Bytes()
	return &bs

	// writer := packet.Writer()
	// //size uint32
	// writer.WriteU32(uint32(len(*data)))
	// //crc32 uint32
	// crc32 := crc32.Checksum(*data, crc32.IEEETable)
	// writer.WriteU32(crc32)

	// //msgID
	// writer.WriteU32(msgID)
	// //Data
	// writer.WriteRawBytes(*data)
	// bs := writer.Data()

	// return &bs
}

func LockPartition(pg *sql.DB, ns string, max uint64) (uint64, error) {
	tab := crc32.MakeTable(crc32.IEEE)
	for {
		var p uint64
		for p = 0; p < max; p++ {
			pId := fmt.Sprintf("%s.%d", ns, p)
			check := crc32.Checksum([]byte(pId), tab)
			rows, err := pg.Query("select pg_try_advisory_lock($1)", check)
			if err != nil {
				continue
			}
			for rows.Next() {
				var result sql.NullBool
				rows.Scan(&result)
				if result.Valid && result.Bool {
					fmt.Printf("at=%q partition=%d max=%d\n",
						"acquired-lock", p, max)
					rows.Close()
					return p, nil
				}
			}
			rows.Close()
		}
		fmt.Printf("at=%q\n", "waiting-for-partition-lock")
		time.Sleep(time.Second * 10)
	}
	return 0, errors.New("Unable to lock partition.")
}

func calcNewCRC32(path string, newHeader *BinaryHeader) (uint32, error) {
	buf := &bytes.Buffer{}
	err := binary.Write(buf, binary.BigEndian, newHeader)
	if err != nil {
		return 0, err
	}

	header := buf.Bytes()
	crc := crc32.Checksum(header[:28], crc32.IEEETable) // before crc32
	crc = crc32.Update(crc, crc32.IEEETable, header[32:])

	f, err := os.Open(path)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	if _, err := f.Seek(int64(len(header)), os.SEEK_SET); err != nil {
		return 0, err
	}

	data := make([]byte, 4*1024*1024)
	for {
		n, err := f.Read(data)
		crc = crc32.Update(crc, crc32.IEEETable, data[:n])
		if err != nil {
			if err != io.EOF {
				return 0, err
			}
			break
		}
	}
	return crc, nil
}

func TestReadRecord(t *testing.T) {
	badInfoRecord := make([]byte, len(infoRecord))
	copy(badInfoRecord, infoRecord)
	badInfoRecord[len(badInfoRecord)-1] = 'a'

	tests := []struct {
		data []byte
		wr   *walpb.Record
		we   error
	}{
		{infoRecord, &walpb.Record{Type: 1, Crc: crc32.Checksum(infoData, crcTable), Data: infoData}, nil},
		{[]byte(""), &walpb.Record{}, io.EOF},
		{infoRecord[:8], &walpb.Record{}, io.ErrUnexpectedEOF},
		{infoRecord[:len(infoRecord)-len(infoData)-8], &walpb.Record{}, io.ErrUnexpectedEOF},
		{infoRecord[:len(infoRecord)-len(infoData)], &walpb.Record{}, io.ErrUnexpectedEOF},
		{infoRecord[:len(infoRecord)-8], &walpb.Record{}, io.ErrUnexpectedEOF},
		{badInfoRecord, &walpb.Record{}, walpb.ErrCRCMismatch},
	}

	rec := &walpb.Record{}
	for i, tt := range tests {
		buf := bytes.NewBuffer(tt.data)
		decoder := newDecoder(ioutil.NopCloser(buf))
		e := decoder.decode(rec)
		if !reflect.DeepEqual(rec, tt.wr) {
			t.Errorf("#%d: block = %v, want %v", i, rec, tt.wr)
		}
		if !reflect.DeepEqual(e, tt.we) {
			t.Errorf("#%d: err = %v, want %v", i, e, tt.we)
		}
		rec = &walpb.Record{}
	}
}

func (c *Connection) parcelValidity(parcel Parcel) uint8 {
	verbose(c.peer.PeerIdent(), "Connection.isValidParcel(%s)", parcel.MessageType())
	crc := crc32.Checksum(parcel.Payload, CRCKoopmanTable)
	switch {
	case parcel.Header.NodeID == NodeID: // We are talking to ourselves!
		parcel.Trace("Connection.isValidParcel()-loopback", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to loopback!: %+v", parcel.Header)
		c.peer.QualityScore = MinumumQualityScore - 50 // Ban ourselves for a week
		return InvalidDisconnectPeer
	case parcel.Header.Network != CurrentNetwork:
		parcel.Trace("Connection.isValidParcel()-network", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong network. Remote: %0x Us: %0x", parcel.Header.Network, CurrentNetwork)
		return InvalidDisconnectPeer
	case parcel.Header.Version < ProtocolVersionMinimum:
		parcel.Trace("Connection.isValidParcel()-version", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong version: %+v", parcel.Header)
		return InvalidDisconnectPeer
	case parcel.Header.Length != uint32(len(parcel.Payload)):
		parcel.Trace("Connection.isValidParcel()-length", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong length: %+v", parcel.Header)
		return InvalidPeerDemerit
	case parcel.Header.Crc32 != crc:
		parcel.Trace("Connection.isValidParcel()-checksum", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to bad checksum: %+v", parcel.Header)
		return InvalidPeerDemerit
	default:
		parcel.Trace("Connection.isValidParcel()-ParcelValid", "H")
		return ParcelValid
	}
	return ParcelValid
}

// Create an object struct for the given attributes and contents.
//
// LOCKS_REQUIRED(b.mu)
func (b *bucket) mintObject(
	req *gcs.CreateObjectRequest,
	contents []byte) (o fakeObject) {
	md5Sum := md5.Sum(contents)

	// Set up basic info.
	b.prevGeneration++
	o.metadata = gcs.Object{
		Name:            req.Name,
		ContentType:     req.ContentType,
		ContentLanguage: req.ContentLanguage,
		CacheControl:    req.CacheControl,
		Owner:           "user-fake",
		Size:            uint64(len(contents)),
		ContentEncoding: req.ContentEncoding,
		ComponentCount:  1,
		MD5:             &md5Sum,
		CRC32C:          crc32.Checksum(contents, crc32cTable),
		MediaLink:       "http://localhost/download/storage/fake/" + req.Name,
		Metadata:        copyMetadata(req.Metadata),
		Generation:      b.prevGeneration,
		MetaGeneration:  1,
		StorageClass:    "STANDARD",
		Updated:         b.clock.Now(),
	}

	// Set up data.
	o.data = contents

	return
}

// sendSerializedData sets the leading CRC on buf and writes the data to conn.  Any errors on the
// write will be logged but otherwise ignored.
func sendSerializedData(buf []byte, conn io.Writer) {
	// Prefix buf with a crc of everything we've appended to it.
	AppendUint32(buf[0:0], crc32.Checksum(buf[4:], crcTable))
	_, err := conn.Write(buf)
	if err != nil {
		log.Printf("Failed to write %d bytes in BatchAndSend: %v", err)
	}
}

// serializeWriteRow serialize the current block file and offset where new
// will be written into a format suitable for storage into the metadata.
func serializeWriteRow(curBlockFileNum, curFileOffset uint32) []byte {
	var serializedRow [12]byte
	byteOrder.PutUint32(serializedRow[0:4], curBlockFileNum)
	byteOrder.PutUint32(serializedRow[4:8], curFileOffset)
	checksum := crc32.Checksum(serializedRow[:8], castagnoli)
	byteOrder.PutUint32(serializedRow[8:12], checksum)
	return serializedRow[:]
}

func Benchmark_crc32_6(b *testing.B) {
	buf, n := initSample(6)
	tab := crc32.MakeTable(crc32.Castagnoli)
	b.SetBytes(n)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		crc32.Checksum(buf, tab)
	}
}