Golang codable.DecodeUint64函数代码示例

func newHeadsScanner(filename string) *headsScanner {
	hs := &headsScanner{}
	defer func() {
		if hs.f != nil && hs.err != nil {
			hs.f.Close()
		}
	}()

	if hs.f, hs.err = os.Open(filename); hs.err != nil {
		return hs
	}
	hs.r = bufio.NewReaderSize(hs.f, fileBufSize)

	buf := make([]byte, len(headsMagicString))
	if _, hs.err = io.ReadFull(hs.r, buf); hs.err != nil {
		return hs
	}
	magic := string(buf)
	if magic != headsMagicString {
		hs.err = fmt.Errorf(
			"unexpected magic string, want %q, got %q",
			headsMagicString, magic,
		)
		return hs
	}
	hs.version, hs.err = binary.ReadVarint(hs.r)
	if (hs.version != headsFormatVersion && hs.version != headsFormatLegacyVersion) || hs.err != nil {
		hs.err = fmt.Errorf(
			"unknown or unreadable heads format version, want %d, got %d, error: %s",
			headsFormatVersion, hs.version, hs.err,
		)
		return hs
	}
	if hs.seriesTotal, hs.err = codable.DecodeUint64(hs.r); hs.err != nil {
		return hs
	}
	return hs
}

// loadSeriesMapAndHeads loads the fingerprint to memory-series mapping and all
// the chunks contained in the checkpoint (and thus not yet persisted to series
// files). The method is capable of loading the checkpoint format v1 and v2. If
// recoverable corruption is detected, or if the dirty flag was set from the
// beginning, crash recovery is run, which might take a while. If an
// unrecoverable error is encountered, it is returned. Call this method during
// start-up while nothing else is running in storage land. This method is
// utterly goroutine-unsafe.
func (p *persistence) loadSeriesMapAndHeads() (sm *seriesMap, chunksToPersist int64, err error) {
	var chunkDescsTotal int64
	fingerprintToSeries := make(map[clientmodel.Fingerprint]*memorySeries)
	sm = &seriesMap{m: fingerprintToSeries}

	defer func() {
		if sm != nil && p.dirty {
			log.Warn("Persistence layer appears dirty.")
			err = p.recoverFromCrash(fingerprintToSeries)
			if err != nil {
				sm = nil
			}
		}
		if err == nil {
			numMemChunkDescs.Add(float64(chunkDescsTotal))
		}
	}()

	f, err := os.Open(p.headsFileName())
	if os.IsNotExist(err) {
		return sm, 0, nil
	}
	if err != nil {
		log.Warn("Could not open heads file:", err)
		p.dirty = true
		return
	}
	defer f.Close()
	r := bufio.NewReaderSize(f, fileBufSize)

	buf := make([]byte, len(headsMagicString))
	if _, err := io.ReadFull(r, buf); err != nil {
		log.Warn("Could not read from heads file:", err)
		p.dirty = true
		return sm, 0, nil
	}
	magic := string(buf)
	if magic != headsMagicString {
		log.Warnf(
			"unexpected magic string, want %q, got %q",
			headsMagicString, magic,
		)
		p.dirty = true
		return
	}
	version, err := binary.ReadVarint(r)
	if (version != headsFormatVersion && version != headsFormatLegacyVersion) || err != nil {
		log.Warnf("unknown heads format version, want %d", headsFormatVersion)
		p.dirty = true
		return sm, 0, nil
	}
	numSeries, err := codable.DecodeUint64(r)
	if err != nil {
		log.Warn("Could not decode number of series:", err)
		p.dirty = true
		return sm, 0, nil
	}

	for ; numSeries > 0; numSeries-- {
		seriesFlags, err := r.ReadByte()
		if err != nil {
			log.Warn("Could not read series flags:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		headChunkPersisted := seriesFlags&flagHeadChunkPersisted != 0
		fp, err := codable.DecodeUint64(r)
		if err != nil {
			log.Warn("Could not decode fingerprint:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		var metric codable.Metric
		if err := metric.UnmarshalFromReader(r); err != nil {
			log.Warn("Could not decode metric:", err)
			p.dirty = true
			return sm, chunksToPersist, nil
		}
		var persistWatermark int64
		var modTime time.Time
		if version != headsFormatLegacyVersion {
			// persistWatermark only present in v2.
			persistWatermark, err = binary.ReadVarint(r)
			if err != nil {
				log.Warn("Could not decode persist watermark:", err)
				p.dirty = true
				return sm, chunksToPersist, nil
			}
			modTimeNano, err := binary.ReadVarint(r)
			if err != nil {
				log.Warn("Could not decode modification time:", err)
				p.dirty = true
//.........这里部分代码省略.........

// loadFPMappings loads the fingerprint mappings. It also returns the highest
// mapped fingerprint and any error encountered. If p.mappingsFileName is not
// found, the method returns (fpMappings{}, 0, nil). Do not call concurrently
// with checkpointFPMappings.
func (p *persistence) loadFPMappings() (fpMappings, clientmodel.Fingerprint, error) {
	fpm := fpMappings{}
	var highestMappedFP clientmodel.Fingerprint

	f, err := os.Open(p.mappingsFileName())
	if os.IsNotExist(err) {
		return fpm, 0, nil
	}
	if err != nil {
		return nil, 0, err
	}
	defer f.Close()
	r := bufio.NewReaderSize(f, fileBufSize)

	buf := make([]byte, len(mappingsMagicString))
	if _, err := io.ReadFull(r, buf); err != nil {
		return nil, 0, err
	}
	magic := string(buf)
	if magic != mappingsMagicString {
		return nil, 0, fmt.Errorf(
			"unexpected magic string, want %q, got %q",
			mappingsMagicString, magic,
		)
	}
	version, err := binary.ReadUvarint(r)
	if version != mappingsFormatVersion || err != nil {
		return nil, 0, fmt.Errorf("unknown fingerprint mappings format version, want %d", mappingsFormatVersion)
	}
	numRawFPs, err := binary.ReadUvarint(r)
	if err != nil {
		return nil, 0, err
	}
	for ; numRawFPs > 0; numRawFPs-- {
		rawFP, err := codable.DecodeUint64(r)
		if err != nil {
			return nil, 0, err
		}
		numMappings, err := binary.ReadUvarint(r)
		if err != nil {
			return nil, 0, err
		}
		mappings := make(map[string]clientmodel.Fingerprint, numMappings)
		for ; numMappings > 0; numMappings-- {
			lenMS, err := binary.ReadUvarint(r)
			if err != nil {
				return nil, 0, err
			}
			buf := make([]byte, lenMS)
			if _, err := io.ReadFull(r, buf); err != nil {
				return nil, 0, err
			}
			fp, err := codable.DecodeUint64(r)
			if err != nil {
				return nil, 0, err
			}
			mappedFP := clientmodel.Fingerprint(fp)
			if mappedFP > highestMappedFP {
				highestMappedFP = mappedFP
			}
			mappings[string(buf)] = mappedFP
		}
		fpm[clientmodel.Fingerprint(rawFP)] = mappings
	}
	return fpm, highestMappedFP, nil
}

// scan works like bufio.Scanner.Scan.
func (hs *headsScanner) scan() bool {
	if hs.seriesCurrent == hs.seriesTotal || hs.err != nil {
		return false
	}

	var (
		seriesFlags      byte
		fpAsInt          uint64
		metric           codable.Metric
		persistWatermark int64
		modTimeNano      int64
		modTime          time.Time
		chunkDescsOffset int64
		savedFirstTime   int64
		numChunkDescs    int64
		firstTime        int64
		lastTime         int64
		encoding         byte
		ch               chunk.Chunk
		lastTimeHead     model.Time
	)
	if seriesFlags, hs.err = hs.r.ReadByte(); hs.err != nil {
		return false
	}
	headChunkPersisted := seriesFlags&flagHeadChunkPersisted != 0
	if fpAsInt, hs.err = codable.DecodeUint64(hs.r); hs.err != nil {
		return false
	}
	hs.fp = model.Fingerprint(fpAsInt)

	if hs.err = metric.UnmarshalFromReader(hs.r); hs.err != nil {
		return false
	}
	if hs.version != headsFormatLegacyVersion {
		// persistWatermark only present in v2.
		persistWatermark, hs.err = binary.ReadVarint(hs.r)
		if hs.err != nil {
			return false
		}
		modTimeNano, hs.err = binary.ReadVarint(hs.r)
		if hs.err != nil {
			return false
		}
		if modTimeNano != -1 {
			modTime = time.Unix(0, modTimeNano)
		}
	}
	if chunkDescsOffset, hs.err = binary.ReadVarint(hs.r); hs.err != nil {
		return false
	}
	if savedFirstTime, hs.err = binary.ReadVarint(hs.r); hs.err != nil {
		return false
	}

	if numChunkDescs, hs.err = binary.ReadVarint(hs.r); hs.err != nil {
		return false
	}
	chunkDescs := make([]*chunk.Desc, numChunkDescs)
	if hs.version == headsFormatLegacyVersion {
		if headChunkPersisted {
			persistWatermark = numChunkDescs
		} else {
			persistWatermark = numChunkDescs - 1
		}
	}
	headChunkClosed := true // Initial assumption.
	for i := int64(0); i < numChunkDescs; i++ {
		if i < persistWatermark {
			if firstTime, hs.err = binary.ReadVarint(hs.r); hs.err != nil {
				return false
			}
			if lastTime, hs.err = binary.ReadVarint(hs.r); hs.err != nil {
				return false
			}
			chunkDescs[i] = &chunk.Desc{
				ChunkFirstTime: model.Time(firstTime),
				ChunkLastTime:  model.Time(lastTime),
			}
			chunk.NumMemDescs.Inc()
		} else {
			// Non-persisted chunk.
			// If there are non-persisted chunks at all, we consider
			// the head chunk not to be closed yet.
			headChunkClosed = false
			if encoding, hs.err = hs.r.ReadByte(); hs.err != nil {
				return false
			}
			if ch, hs.err = chunk.NewForEncoding(chunk.Encoding(encoding)); hs.err != nil {
				return false
			}
			if hs.err = ch.Unmarshal(hs.r); hs.err != nil {
				return false
			}
			cd := chunk.NewDesc(ch, ch.FirstTime())
			if i < numChunkDescs-1 {
				// This is NOT the head chunk. So it's a chunk
				// to be persisted, and we need to populate lastTime.
				hs.chunksToPersistTotal++
				cd.MaybePopulateLastTime()
//.........这里部分代码省略.........

// loadSeriesMapAndHeads loads the fingerprint to memory-series mapping and all
// open (non-full) head chunks. If recoverable corruption is detected, or if the
// dirty flag was set from the beginning, crash recovery is run, which might
// take a while. If an unrecoverable error is encountered, it is returned. Call
// this method during start-up while nothing else is running in storage
// land. This method is utterly goroutine-unsafe.
func (p *persistence) loadSeriesMapAndHeads() (sm *seriesMap, err error) {
	var chunksTotal, chunkDescsTotal int64
	fingerprintToSeries := make(map[clientmodel.Fingerprint]*memorySeries)
	sm = &seriesMap{m: fingerprintToSeries}

	defer func() {
		if sm != nil && p.dirty {
			glog.Warning("Persistence layer appears dirty.")
			err = p.recoverFromCrash(fingerprintToSeries)
			if err != nil {
				sm = nil
			}
		}
		if err == nil {
			atomic.AddInt64(&numMemChunks, chunksTotal)
			numMemChunkDescs.Add(float64(chunkDescsTotal))
		}
	}()

	f, err := os.Open(p.headsFileName())
	if os.IsNotExist(err) {
		return sm, nil
	}
	if err != nil {
		glog.Warning("Could not open heads file:", err)
		p.dirty = true
		return
	}
	defer f.Close()
	r := bufio.NewReaderSize(f, fileBufSize)

	buf := make([]byte, len(headsMagicString))
	if _, err := io.ReadFull(r, buf); err != nil {
		glog.Warning("Could not read from heads file:", err)
		p.dirty = true
		return sm, nil
	}
	magic := string(buf)
	if magic != headsMagicString {
		glog.Warningf(
			"unexpected magic string, want %q, got %q",
			headsMagicString, magic,
		)
		p.dirty = true
		return
	}
	if version, err := binary.ReadVarint(r); version != headsFormatVersion || err != nil {
		glog.Warningf("unknown heads format version, want %d", headsFormatVersion)
		p.dirty = true
		return sm, nil
	}
	numSeries, err := codable.DecodeUint64(r)
	if err != nil {
		glog.Warning("Could not decode number of series:", err)
		p.dirty = true
		return sm, nil
	}

	for ; numSeries > 0; numSeries-- {
		seriesFlags, err := r.ReadByte()
		if err != nil {
			glog.Warning("Could not read series flags:", err)
			p.dirty = true
			return sm, nil
		}
		headChunkPersisted := seriesFlags&flagHeadChunkPersisted != 0
		fp, err := codable.DecodeUint64(r)
		if err != nil {
			glog.Warning("Could not decode fingerprint:", err)
			p.dirty = true
			return sm, nil
		}
		var metric codable.Metric
		if err := metric.UnmarshalFromReader(r); err != nil {
			glog.Warning("Could not decode metric:", err)
			p.dirty = true
			return sm, nil
		}
		chunkDescsOffset, err := binary.ReadVarint(r)
		if err != nil {
			glog.Warning("Could not decode chunk descriptor offset:", err)
			p.dirty = true
			return sm, nil
		}
		savedFirstTime, err := binary.ReadVarint(r)
		if err != nil {
			glog.Warning("Could not decode saved first time:", err)
			p.dirty = true
			return sm, nil
		}
		numChunkDescs, err := binary.ReadVarint(r)
		if err != nil {
			glog.Warning("Could not decode number of chunk descriptors:", err)
			p.dirty = true
//.........这里部分代码省略.........