Golang adler32.Checksum函数代码示例

func CheckECC(data []byte) uint32 {
	if len(data) > 200 {
		tmpdata := append(data[:100], data[len(data)-100:]...)
		return adler32.Checksum(tmpdata)
	} else {
		return adler32.Checksum(data)
	}
}

// FIXME: find better way of writing this
func Encode(msg *RpcMessage) ([]byte, error) {
	payload, err := proto.Marshal(msg)
	if err != nil {
		return nil, err
	}

	wire := make([]byte, 12+len(payload))

	// size
	binary.BigEndian.PutUint32(wire, uint32(8+len(payload)))

	// marker
	if copy(wire[4:], "RPC0") != 4 {
		panic("What the hell")
	}

	// payload
	if copy(wire[8:], payload) != len(payload) {
		panic("What the hell")
	}

	// checksum
	checksum := adler32.Checksum(wire[4 : 8+len(payload)])
	binary.BigEndian.PutUint32(wire[8+len(payload):], checksum)

	return wire, nil
}

func BenchmarkAdler32(b *testing.B) {
	var bv uint32
	for i := 0; i < b.N; i++ {
		bv = adler32.Checksum(in)
	}
	benchVal32 = bv
}

func (this *StandardPeerSelector) PickPeer(key string) (peerAddr string) {
	// adler32 is almost same as crc32, but much 3 times faster
	checksum := adler32.Checksum([]byte(key))
	index := int(checksum) % len(this.peerAddrs)

	return this.peerAddrs[index]
}

// performReducing runs the reducing goroutines.
func performReducing(mr MapReducer, mapEmitChan, reduceEmitChan KeyValueChan) {
	// Start a closer for the reduce emit chan.
	size := runtime.NumCPU()
	signals := newCloserChan(reduceEmitChan, size)

	// Start reduce goroutines.
	reduceChans := make([]KeyValueChan, size)
	for i := 0; i < size; i++ {
		reduceChans[i] = make(KeyValueChan)
		go func(in KeyValueChan) {
			mr.Reduce(in, reduceEmitChan)
			signals <- struct{}{}
		}(reduceChans[i])
	}

	// Read map emitted data.
	for kv := range mapEmitChan {
		hash := adler32.Checksum([]byte(kv.Key()))
		idx := hash % uint32(size)
		reduceChans[idx] <- kv
	}

	// Close reduce channels.
	for _, reduceChan := range reduceChans {
		reduceChan.Close()
	}
}

// NewReaderDict is like NewReader but uses a preset dictionary.
// NewReaderDict ignores the dictionary if the compressed data does not refer to it.
func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, os.Error) {
	z := new(reader)
	if fr, ok := r.(flate.Reader); ok {
		z.r = fr
	} else {
		z.r = bufio.NewReader(r)
	}
	_, err := io.ReadFull(z.r, z.scratch[0:2])
	if err != nil {
		return nil, err
	}
	h := uint(z.scratch[0])<<8 | uint(z.scratch[1])
	if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) {
		return nil, HeaderError
	}
	if z.scratch[1]&0x20 != 0 {
		_, err = io.ReadFull(z.r, z.scratch[0:4])
		if err != nil {
			return nil, err
		}
		checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
		if checksum != adler32.Checksum(dict) {
			return nil, DictionaryError
		}
		z.decompressor = flate.NewReaderDict(z.r, dict)
	} else {
		z.decompressor = flate.NewReader(z.r)
	}
	z.digest = adler32.New()
	return z, nil
}

func Decode(r io.Reader) (msg *RpcMessage, err error) {
	header := make([]byte, 4)
	_, err = io.ReadFull(r, header)
	if err != nil {
		return
	}

	length := binary.BigEndian.Uint32(header)
	payload := make([]byte, length)
	_, err = io.ReadFull(r, payload)
	if err != nil {
		return
	}

	if string(payload[:4]) != "RPC0" {
		err = fmt.Errorf("Wrong marker")
		return
	}

	checksum := adler32.Checksum(payload[:length-4])
	if checksum != binary.BigEndian.Uint32(payload[length-4:]) {
		err = fmt.Errorf("Wrong checksum")
		return
	}

	msg = new(RpcMessage)
	err = proto.Unmarshal(payload[4:length-4], msg)
	return
}

// Perform the reducing.
func performReducing(mapEmitChan KeyValueChan, reduceFunc ReduceFunc, reduceSize int, reduceEmitChan KeyValueChan) {
	// Start a closer for the reduce emit chan.

	sigChan := closeSignalChannel(reduceEmitChan, reduceSize)

	// Start reduce funcs.

	reduceChans := make(KeyValueChans, reduceSize)

	for i := 0; i < reduceSize; i++ {
		reduceChans[i] = make(KeyValueChan)

		go func(inChan KeyValueChan) {
			reduceFunc(inChan, reduceEmitChan)

			sigChan <- true
		}(reduceChans[i])
	}

	// Read map emitted data.

	for kv := range mapEmitChan {
		hash := adler32.Checksum([]byte(kv.Key))
		idx := hash % uint32(reduceSize)

		reduceChans[idx] <- kv
	}

	// Close reduce channels.

	for _, reduceChan := range reduceChans {
		close(reduceChan)
	}
}

func (section_header *Section_Header) Verify(datar *bytes.Reader) bool {
	var buf []byte

	datar.Read(buf)
	fmt.Println(section_header.Checksum, len(buf))
	return section_header.Checksum == adler32.Checksum(buf[:72])

}

func errorClass(err error) string {
	class := reflect.TypeOf(err).String()
	if class == "" {
		return "panic"
	} else if class == "*errors.errorString" {
		checksum := adler32.Checksum([]byte(err.Error()))
		return fmt.Sprintf("{%x}", checksum)
	} else {
		return strings.TrimPrefix(class, "*")
	}
}

// Put sets a value by key in the hash map
func (t *HashTable) Put(key, value string) {
	hash := adler32.Checksum([]byte(key)) % maxTableSize
	for {
		if t.Table[hash] != nil && t.Table[hash].Key != key {
			hash = (hash + 1) % maxTableSize
		} else {
			break
		}
	}
	t.Table[hash] = &node{key, value}
}

func (c *Catalog) Include(content []byte) (ret bool) {
	crc := adler32.Checksum([]byte(content))
	sort.Sort(c.Files)
	exists := sort.Search(len(c.Files), func(i int) bool {
		return c.Files[i] >= crc
	})

	if exists < len(c.Files) && c.Files[exists] == crc {
		return true
	}
	return false
}

func BenchmarkAdler32(b *testing.B) {
	in := make([]byte, 10000)
	for i := 0; i < len(in); i++ {
		in[i] = byte(rand.Intn(255))
	}
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		adler32.Checksum(in)
	}
	b.SetBytes(int64(len(in)))
}

// writeHeader writes the ZLIB header.
func (z *Writer) writeHeader() (err error) {
	z.wroteHeader = true
	// ZLIB has a two-byte header (as documented in RFC 1950).
	// The first four bits is the CINFO (compression info), which is 7 for the default deflate window size.
	// The next four bits is the CM (compression method), which is 8 for deflate.
	z.scratch[0] = 0x78
	// The next two bits is the FLEVEL (compression level). The four values are:
	// 0=fastest, 1=fast, 2=default, 3=best.
	// The next bit, FDICT, is set if a dictionary is given.
	// The final five FCHECK bits form a mod-31 checksum.
	switch z.level {
	case 0, 1:
		z.scratch[1] = 0 << 6
	case 2, 3, 4, 5:
		z.scratch[1] = 1 << 6
	case 6, -1:
		z.scratch[1] = 2 << 6
	case 7, 8, 9:
		z.scratch[1] = 3 << 6
	default:
		panic("unreachable")
	}
	if z.dict != nil {
		z.scratch[1] |= 1 << 5
	}
	z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31)
	if _, err = z.w.Write(z.scratch[0:2]); err != nil {
		return err
	}
	if z.dict != nil {
		// The next four bytes are the Adler-32 checksum of the dictionary.
		checksum := adler32.Checksum(z.dict)
		z.scratch[0] = uint8(checksum >> 24)
		z.scratch[1] = uint8(checksum >> 16)
		z.scratch[2] = uint8(checksum >> 8)
		z.scratch[3] = uint8(checksum >> 0)
		if _, err = z.w.Write(z.scratch[0:4]); err != nil {
			return err
		}
	}
	if z.compressor == nil {
		// Initialize deflater unless the Writer is being reused
		// after a Reset call.
		z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict)
		if err != nil {
			return err
		}
		z.digest = adler32.New()
	}
	return nil
}

// StartJob launches a job on the given queue. It is not executed immediately but
// scheduled to run as a task which performs splitting of the input reader based
// on the number of shards.
func (m *mapper) startJobHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	values := r.URL.Query()
	name := values.Get("name")
	jobSpec, err := CreateJobInstance(name)
	if err != nil {
		return
	}

	shards, err := strconv.Atoi(values.Get("shards"))
	if shards == 0 || err != nil {
		shards = m.config.Shards
	}

	queue := values.Get("queue")
	if queue != "" {
		// override the queue for this request
		// (used by locker.Schedule later)
		c = locker.WithQueue(c, queue)
	}
	bucket := values.Get("bucket")

	query, err := jobSpec.Query(r)
	if err != nil {
		log.Errorf(c, "error creating query %v", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	requestHash := r.Header.Get("X-Appengine-Request-Id-Hash")
	if requestHash == "" {
		// this should only happen when testing, we just need a short hash
		requestID := appengine.RequestID(c)
		requestHash = strconv.FormatUint(uint64(adler32.Checksum([]byte(requestID))), 16)
	}

	id := fmt.Sprintf("%s/%s", name, requestHash)
	job := &job{
		JobName:   name,
		JobSpec:   jobSpec,
		Bucket:    bucket,
		Shards:    shards,
		Iterating: true,
	}
	job.common.start(query)

	key := datastore.NewKey(c, m.config.DatastorePrefix+jobKind, id, 0, nil)
	m.locker.Schedule(c, key, job, m.config.Path+jobURL, nil)
}