Golang needle.NewCache函数代码示例

// init recovery super block from index or super block.
func (v *Volume) init() (err error) {
	var offset uint32
	// recovery from index
	if err = v.Indexer.Recovery(func(ix *index.Index) (err1 error) {
		v.needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		offset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		if ix.Size > int32(v.conf.NeedleMaxSize) || ix.Size < 0 {
			err1 = errors.ErrIndexSize
		}
		return
	}); err != nil {
		return
	}
	// recovery from super block
	err = v.Block.Recovery(offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if n.TotalSize > int32(v.conf.NeedleMaxSize) || n.TotalSize < 0 {
			err1 = errors.ErrNeedleSize
			return
		}
		if n.Flag == needle.FlagOK {
			if err1 = v.Indexer.Write(n.Key, so, n.TotalSize); err1 != nil {
				return
			}
		} else {
			so = needle.CacheDelOffset
		}
		v.needles[n.Key] = needle.NewCache(so, n.TotalSize)
		return
	})
	// flush index
	err = v.Indexer.Flush()
	return
}

// init recovery super block from index or super block.
func (v *Volume) init() (err error) {
	var (
		size               int64
		lastOffset, offset uint32
	)
	// recovery from index
	if err = v.Indexer.Recovery(func(ix *index.Index) error {
		if ix.Size > int32(v.conf.NeedleMaxSize) || ix.Size < 0 {
			log.Error("recovery index: %s error(%v)", ix, errors.ErrIndexSize)
			return errors.ErrIndexSize
		}
		// must no less than last offset
		if ix.Offset < lastOffset {
			log.Error("recovery index: %s lastoffset: %d error(%v)", ix, lastOffset, errors.ErrIndexOffset)
			return errors.ErrIndexOffset
		}
		// WARN if index's offset more than the block, discard it.
		if size = int64(ix.Size) + needle.BlockOffset(ix.Offset); size > v.Block.Size {
			log.Error("recovery index: %s EOF", ix)
			return errors.ErrIndexEOF
		}
		v.needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		offset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		lastOffset = ix.Offset
		return nil
	}); err != nil && err != errors.ErrIndexEOF {
		return
	}
	// recovery from super block
	if err = v.Block.Recovery(offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if n.TotalSize > int32(v.conf.NeedleMaxSize) || n.TotalSize < 0 {
			log.Error("recovery needle: %s error(%v)", n, errors.ErrNeedleSize)
			return errors.ErrNeedleSize
		}
		if n.Flag == needle.FlagOK {
			if err1 = v.Indexer.Write(n.Key, so, n.TotalSize); err1 != nil {
				return
			}
		} else {
			so = needle.CacheDelOffset
		}
		v.needles[n.Key] = needle.NewCache(so, n.TotalSize)
		return
	}); err != nil {
		return
	}
	// recheck offset, keep size and offset consistency
	if v.Block.Size != needle.BlockOffset(v.Block.Offset) {
		log.Error("block: %s size: %d, offset: %d (%d size) not consistency", v.Block.File, v.Block.Size, v.Block.Offset, needle.BlockOffset(v.Block.Offset))
		return errors.ErrSuperBlockOffset
	}
	// flush index
	err = v.Indexer.Flush()
	return
}

// Del logical delete a needle, update disk needle flag and memory needle
// cache offset to zero.
func (v *Volume) Del(key int64) (err error) {
	var (
		ok     bool
		nc     int64
		size   int32
		offset uint32
	)
	v.lock.Lock()
	nc, ok = v.needles[key]
	if ok {
		if offset, size = needle.Cache(nc); offset != needle.CacheDelOffset {
			v.needles[key] = needle.NewCache(needle.CacheDelOffset, size)
			// when in compact, must save all del operations.
			if v.Compact {
				v.compactKeys = append(v.compactKeys, key)
			}
		} else {
			err = errors.ErrNeedleDeleted
		}
	} else {
		err = errors.ErrNeedleNotExist
	}
	v.lock.Unlock()
	if err == nil {
		err = v.asyncDel(offset)
	}
	return
}

// Add add a needles, if key exists append to super block, then update
// needle cache offset to new offset.
func (v *Volume) Add(n *needle.Needle) (err error) {
	var (
		ok              bool
		nc              int64
		offset, ooffset uint32
		now             = time.Now().UnixNano()
	)
	v.lock.Lock()
	offset = v.Block.Offset
	if err = v.Block.Write(n.Buffer[:n.TotalSize]); err == nil {
		if err = v.Indexer.Add(n.Key, offset, n.TotalSize); err == nil {
			nc, ok = v.needles[n.Key]
			v.needles[n.Key] = needle.NewCache(offset, n.TotalSize)
			v.addCheck(n.Key, n.Cookie)
		}
	}
	v.lock.Unlock()
	if err != nil {
		return
	}
	if log.V(1) {
		log.Infof("add needle, offset: %d, size: %d", offset, n.TotalSize)
		log.Info(n)
	}
	if ok {
		ooffset, _ = needle.Cache(nc)
		err = v.asyncDel(ooffset)
		log.Warningf("same key: %d, old offset: %d, new offset: %d", n.Key, ooffset, offset)
	}
	atomic.AddUint64(&v.Stats.TotalWriteProcessed, 1)
	atomic.AddUint64(&v.Stats.TotalWriteBytes, uint64(n.TotalSize))
	atomic.AddUint64(&v.Stats.TotalWriteDelay, uint64(time.Now().UnixNano()-now))
	return
}

// Get get a needle by key and cookie.
func (v *Volume) Get(n *needle.Needle) (err error) {
	var (
		ok     bool
		nc     int64
		size   int32
		offset uint32
		key    = n.Key
		cookie = n.Cookie
		now    = time.Now().UnixNano()
	)
	// WARN pread syscall is atomic, so use rlock
	v.lock.RLock()
	if nc, ok = v.needles[n.Key]; ok {
		offset, size = needle.Cache(nc)
		if offset != needle.CacheDelOffset {
			err = v.Block.Get(offset, n.Buffer[:size])
		} else {
			err = errors.ErrNeedleDeleted
		}
	} else {
		err = errors.ErrNeedleNotExist
	}
	v.lock.RUnlock()
	if err != nil {
		return
	}
	if log.V(1) {
		log.Infof("get needle key: %d, cookie: %d, offset: %d, size: %d", n.Key, n.Cookie, offset, size)
	}
	if err = n.Parse(); err != nil {
		return
	}
	if log.V(1) {
		log.Infof("%v\n", n)
	}
	if n.Key != key {
		err = errors.ErrNeedleKey
		return
	}
	if n.Cookie != cookie {
		err = errors.ErrNeedleCookie
		return
	}
	// needles map may be out-dated, recheck
	if n.Flag == needle.FlagDel {
		v.lock.Lock()
		v.needles[key] = needle.NewCache(needle.CacheDelOffset, size)
		v.lock.Unlock()
		err = errors.ErrNeedleDeleted
	} else {
		atomic.AddUint64(&v.Stats.TotalGetProcessed, 1)
		atomic.AddUint64(&v.Stats.TotalReadBytes, uint64(size))
		atomic.AddUint64(&v.Stats.TotalGetDelay, uint64(time.Now().UnixNano()-now))
	}
	return
}

// Write write needles, if key exists append to super block, then update
// needle cache offset to new offset.
func (v *Volume) Write(ns []needle.Needle, buf []byte) (err error) {
	var (
		now             = time.Now().UnixNano()
		i               int
		ok              bool
		nc              int64
		ncs             []int64
		offset, ooffset uint32
		ts              int32
		n               *needle.Needle
	)
	if v.Debug {
		for i = 0; i < len(ns); i++ {
			n = &ns[i]
			ts += n.TotalSize
		}
		if int(ts) != len(buf) {
			err = errors.ErrNeedleSize
			return
		}
	}
	v.lock.Lock()
	offset = v.Block.Offset
	if err = v.Block.Write(buf); err == nil {
		for i = 0; i < len(ns); i++ {
			n = &ns[i]
			if err = v.Indexer.Add(n.Key, offset, n.TotalSize); err != nil {
				break
			}
			if nc, ok = v.needles[n.Key]; ok {
				ncs = append(ncs, nc)
			}
			v.needles[n.Key] = needle.NewCache(offset, n.TotalSize)
			v.addCheck(n.Key, n.Cookie)
			offset += n.IncrOffset
			if log.V(1) {
				log.Infof("add needle, offset: %d, size: %d", offset, n.TotalSize)
			}
		}
	}
	v.lock.Unlock()
	if err != nil {
		return
	}
	for _, nc = range ncs {
		ooffset, _ = needle.Cache(nc)
		err = v.asyncDel(ooffset)
		log.Warningf("same key: %d, old offset: %d, new offset: %d", n.Key, ooffset, offset)
	}
	atomic.AddUint64(&v.Stats.TotalWriteProcessed, 1)
	atomic.AddUint64(&v.Stats.TotalWriteBytes, uint64(n.TotalSize))
	atomic.AddUint64(&v.Stats.TotalWriteDelay, uint64(time.Now().UnixNano()-now))
	return
}

// Write write needles, if key exists append to super block, then update
// needle cache offset to new offset.
func (v *Volume) Write(ns *needle.Needles) (err error) {
	var (
		i               int
		ok              bool
		nc              int64
		ncs             []int64
		offset, ooffset uint32
		n               *needle.Needle
		now             = time.Now().UnixNano()
	)
	v.lock.Lock()
	offset = v.Block.Offset
	if err = v.Block.Write(ns.Buffer()); err == nil {
		for i = 0; i < ns.Num; i++ {
			n = ns.Needle(i)
			if err = v.Indexer.Add(n.Key, offset, n.TotalSize); err != nil {
				break
			}
			if nc, ok = v.needles[n.Key]; ok {
				ncs = append(ncs, nc)
			}
			v.needles[n.Key] = needle.NewCache(offset, n.TotalSize)
			v.addCheck(n.Key, n.Cookie)
			offset += n.IncrOffset
			if log.V(1) {
				log.Infof("add needle, offset: %d, size: %d", offset, n.TotalSize)
				log.Info(n)
			}
		}
	}
	v.lock.Unlock()
	if err != nil {
		return
	}
	for _, nc = range ncs {
		ooffset, _ = needle.Cache(nc)
		err = v.asyncDel(ooffset)
		log.Warningf("same key: %d, old offset: %d, new offset: %d", n.Key, ooffset, offset)
	}
	atomic.AddUint64(&v.Stats.TotalWriteProcessed, 1)
	atomic.AddUint64(&v.Stats.TotalWriteBytes, uint64(n.TotalSize))
	atomic.AddUint64(&v.Stats.TotalWriteDelay, uint64(time.Now().UnixNano()-now))
	return
}

//.........这里部分代码省略.........
	// test del, del first needles
	if err = b.Del(1); err != nil {
		t.Errorf("Del() error(%v)", err)
		t.FailNow()
	}
	// test get
	if err = b.Get(1, buf); err != nil {
		t.Errorf("Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 1, 1, needle.FlagDel, n, data, buf); err != nil {
		t.FailNow()
	}
	if err = b.Get(11, buf); err != nil {
		t.Errorf("Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data, buf); err != nil {
		t.FailNow()
	}
	if err = b.Get(16, buf); err != nil {
		t.Errorf("b.Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 4, 4, needle.FlagOK, n, data, buf); err != nil {
		t.FailNow()
	}
	// test recovery
	offset = b.Offset
	if err = b.Recovery(0, func(rn *needle.Needle, so, eo uint32) (err1 error) {
		if rn.Flag != needle.FlagOK {
			so = needle.CacheDelOffset
		}
		needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
		return
	}); err != nil {
		t.Errorf("Recovery() error(%v)", err)
		t.FailNow()
	}
	if b.Offset != offset {
		err = fmt.Errorf("b.Offset not match %d", b.Offset)
		t.Error(err)
		t.FailNow()
	}
	if o, s := needle.Cache(needles[1]); o != needle.CacheDelOffset && s != 40 {
		t.Error("needle.Cache() not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
		t.Error("needle.Cache() not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
		t.Error("needle.Cache() not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
		t.Error("needle.Cache() not match")
		t.FailNow()
	}
	needles = make(map[int64]int64)
	if err = b.Recovery(v2, func(rn *needle.Needle, so, eo uint32) (err1 error) {
		if rn.Flag != needle.FlagOK {
			so = needle.CacheDelOffset
		}
		needles[rn.Key] = needle.NewCache(so, rn.TotalSize)

func TestIndex(t *testing.T) {
	var (
		i       *Indexer
		err     error
		noffset uint32
		file    = "../test/test.idx"
		needles = make(map[int64]int64)
	)
	os.Remove(file)
	defer os.Remove(file)
	if i, err = NewIndexer(file, Options{
		BufferSize:    4 * 1024 * 1024,
		MergeAtTime:   10 * time.Second,
		MergeAtWrite:  5,
		RingBuffer:    10,
		SyncAtWrite:   10,
		Syncfilerange: true,
		NeedleMaxSize: 4 * 1024 * 1024,
	}); err != nil {
		t.Errorf("NewIndexer() error(%v)", err)
		t.FailNow()
	}
	i.Close()
	// test closed
	if err = i.Add(1, 1, 8); err != errors.ErrIndexClosed {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	// test open
	if err = i.Open(); err != nil {
		t.Errorf("Open() error(%v)", err)
		t.FailNow()
	}
	defer i.Close()
	// test add
	if err = i.Add(1, 1, 8); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = i.Add(2, 2, 8); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = i.Add(5, 3, 8); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = i.Add(6, 4, 8); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	i.Signal()
	time.Sleep(1 * time.Second)
	i.Flush()
	// test recovery
	if err = i.Recovery(func(ix *Index) error {
		needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		noffset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		return nil
	}); err != nil {
		t.Errorf("Recovery() error(%v)", err)
		t.FailNow()
	}
	// add 4 index, start with 5
	if noffset != 5 {
		t.Errorf("noffset: %d not match", noffset)
		t.FailNow()
	}
	if o, s := needle.Cache(needles[1]); o != 1 && s != 8 {
		t.Error("needle cache not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[2]); o != 2 && s != 8 {
		t.Error("needle cache not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[5]); o != 3 && s != 8 {
		t.Error("needle cache not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[6]); o != 4 && s != 8 {
		t.Error("needle cache not match")
		t.FailNow()
	}
	// test write
	if err = i.Write(10, 5, 8); err != nil {
		t.Error("Write() error(%v)", err)
		t.FailNow()
	}
	if err = i.Flush(); err != nil {
		t.Error("Flush() error(%v)", err)
		t.FailNow()
	}
	// test recovery
	noffset = 0
	if err = i.Recovery(func(ix *Index) error {
		needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		noffset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		return nil
	}); err != nil {
//.........这里部分代码省略.........