Golang needle.NeedleOffset函数代码示例

// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var (
		data   []byte
		so, eo uint32
		n      = &needle.Needle{}
		rd     = bufio.NewReaderSize(r, b.bufSize)
	)
	if offset == 0 {
		offset = needle.NeedleOffset(headerOffset)
	}
	so, eo = offset, offset
	log.Infof("scan block: %s from offset: %d", b.File, offset)
	if _, err = r.Seek(needle.BlockOffset(offset), os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File)
		return
	}
	for {
		if data, err = rd.Peek(needle.HeaderSize); err != nil {
			break
		}
		if err = n.ParseHeader(data); err != nil {
			break
		}
		if _, err = rd.Discard(needle.HeaderSize); err != nil {
			break
		}
		if data, err = rd.Peek(n.DataSize); err != nil {
			break
		}
		if err = n.ParseData(data); err != nil {
			break
		}
		if _, err = rd.Discard(n.DataSize); err != nil {
			break
		}
		if log.V(1) {
			log.Info(n.String())
		}
		eo += needle.NeedleOffset(int64(n.TotalSize))
		if err = fn(n, so, eo); err != nil {
			break
		}
		so = eo
	}
	if err == io.EOF {
		log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
		err = nil
	} else {
		log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
	}
	return
}

// init init block file, add/parse meta info.
func (b *SuperBlock) init() (err error) {
	var stat os.FileInfo
	if stat, err = b.r.Stat(); err != nil {
		log.Errorf("block: %s Stat() error(%v)", b.File, err)
		return
	}
	if b.Size = stat.Size(); b.Size == 0 {
		// falloc(FALLOC_FL_KEEP_SIZE)
		if err = myos.Fallocate(b.w.Fd(), myos.FALLOC_FL_KEEP_SIZE, 0, _maxSize); err != nil {
			log.Errorf("block: %s Fallocate() error(%s)", b.File, err)
			return
		}
		if err = b.writeMeta(); err != nil {
			log.Errorf("block: %s writeMeta() error(%v)", b.File, err)
			return
		}
		b.Size = _headerSize
	} else {
		if err = b.parseMeta(); err != nil {
			log.Errorf("block: %s parseMeta() error(%v)", b.File, err)
			return
		}
		if _, err = b.w.Seek(_headerOffset, os.SEEK_SET); err != nil {
			log.Errorf("block: %s Seek() error(%v)", b.File, err)
			return
		}
	}
	b.Offset = needle.NeedleOffset(_headerOffset)
	return
}

// Recovery recovery needles map from super block.
func (b *SuperBlock) Recovery(offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	// WARN block may be no left data, must update block offset first
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	b.Offset = offset
	if err = b.Scan(b.r, offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if err1 = fn(n, so, eo); err1 == nil {
			b.Offset = eo
		}
		return
	}); err != nil {
		return
	}
	// advise random read
	// POSIX_FADV_RANDOM disables file readahead entirely.
	// These changes affect the entire file, not just the specified region
	// (but other open file handles to the same file are unaffected).
	if err = myos.Fadvise(b.r.Fd(), 0, 0, myos.POSIX_FADV_RANDOM); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	// reset b.w offset, discard left space which can't parse to a needle
	if _, err = b.w.Seek(needle.BlockOffset(b.Offset), os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File, err)
	}
	return
}

// init recovery super block from index or super block.
func (v *Volume) init() (err error) {
	var offset uint32
	// recovery from index
	if err = v.Indexer.Recovery(func(ix *index.Index) (err1 error) {
		v.needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		offset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		if ix.Size > int32(v.conf.NeedleMaxSize) || ix.Size < 0 {
			err1 = errors.ErrIndexSize
		}
		return
	}); err != nil {
		return
	}
	// recovery from super block
	err = v.Block.Recovery(offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if n.TotalSize > int32(v.conf.NeedleMaxSize) || n.TotalSize < 0 {
			err1 = errors.ErrNeedleSize
			return
		}
		if n.Flag == needle.FlagOK {
			if err1 = v.Indexer.Write(n.Key, so, n.TotalSize); err1 != nil {
				return
			}
		} else {
			so = needle.CacheDelOffset
		}
		v.needles[n.Key] = needle.NewCache(so, n.TotalSize)
		return
	})
	// flush index
	err = v.Indexer.Flush()
	return
}

// NewSuperBlock creae a new super block.
func NewSuperBlock(file string, options Options) (b *SuperBlock, err error) {
	b = &SuperBlock{}
	b.File = file
	b.Options = options
	b.Offset = needle.NeedleOffset(headerSize)
	b.closed = false
	b.write = 0
	b.syncOffset = 0
	b.Padding = needle.PaddingSize
	b.buf = make([]byte, options.BufferSize)
	if b.w, err = os.OpenFile(file, os.O_WRONLY|os.O_CREATE|myos.O_NOATIME, 0664); err != nil {
		log.Errorf("os.OpenFile(\"%s\") error(%v)", file, err)
		b.Close()
		return nil, err
	}
	if b.r, err = os.OpenFile(file, os.O_RDONLY|myos.O_NOATIME, 0664); err != nil {
		log.Errorf("os.OpenFile(\"%s\") error(%v)", file, err)
		b.Close()
		return nil, err
	}
	b.bw = bufio.NewWriterSize(b.w, options.BufferSize)
	if err = b.init(); err != nil {
		log.Errorf("block: %s init() error(%v)", file, err)
		b.Close()
		return nil, err
	}
	return
}

// NewSuperBlock creae a new super block.
func NewSuperBlock(file string, buf int) (b *SuperBlock, err error) {
	b = &SuperBlock{}
	b.closed = false
	b.File = file
	b.bufSize = buf
	b.Offset = needle.NeedleOffset(headerSize)
	b.buf = make([]byte, buf)
	if b.w, err = os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0664); err != nil {
		log.Errorf("os.OpenFile(\"%s\") error(%v)", file, err)
		b.Close()
		return nil, err
	}
	if b.r, err = os.OpenFile(file, os.O_RDONLY, 0664); err != nil {
		log.Errorf("os.OpenFile(\"%s\") error(%v)", file, err)
		b.Close()
		return nil, err
	}
	b.bw = bufio.NewWriterSize(b.w, buf)
	if err = b.init(); err != nil {
		log.Errorf("block: %s init() error(%v)", file, err)
		b.Close()
		return nil, err
	}
	return
}

// init recovery super block from index or super block.
func (v *Volume) init() (err error) {
	var (
		size               int64
		lastOffset, offset uint32
	)
	// recovery from index
	if err = v.Indexer.Recovery(func(ix *index.Index) error {
		if ix.Size > int32(v.conf.NeedleMaxSize) || ix.Size < 0 {
			log.Error("recovery index: %s error(%v)", ix, errors.ErrIndexSize)
			return errors.ErrIndexSize
		}
		// must no less than last offset
		if ix.Offset < lastOffset {
			log.Error("recovery index: %s lastoffset: %d error(%v)", ix, lastOffset, errors.ErrIndexOffset)
			return errors.ErrIndexOffset
		}
		// WARN if index's offset more than the block, discard it.
		if size = int64(ix.Size) + needle.BlockOffset(ix.Offset); size > v.Block.Size {
			log.Error("recovery index: %s EOF", ix)
			return errors.ErrIndexEOF
		}
		v.needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		offset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		lastOffset = ix.Offset
		return nil
	}); err != nil && err != errors.ErrIndexEOF {
		return
	}
	// recovery from super block
	if err = v.Block.Recovery(offset, func(n *needle.Needle, so, eo uint32) (err1 error) {
		if n.TotalSize > int32(v.conf.NeedleMaxSize) || n.TotalSize < 0 {
			log.Error("recovery needle: %s error(%v)", n, errors.ErrNeedleSize)
			return errors.ErrNeedleSize
		}
		if n.Flag == needle.FlagOK {
			if err1 = v.Indexer.Write(n.Key, so, n.TotalSize); err1 != nil {
				return
			}
		} else {
			so = needle.CacheDelOffset
		}
		v.needles[n.Key] = needle.NewCache(so, n.TotalSize)
		return
	}); err != nil {
		return
	}
	// recheck offset, keep size and offset consistency
	if v.Block.Size != needle.BlockOffset(v.Block.Offset) {
		log.Error("block: %s size: %d, offset: %d (%d size) not consistency", v.Block.File, v.Block.Size, v.Block.Offset, needle.BlockOffset(v.Block.Offset))
		return errors.ErrSuperBlockOffset
	}
	// flush index
	err = v.Indexer.Flush()
	return
}

// Write start add needles to the block, must called after start a transaction.
func (b *SuperBlock) Write(n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	var incrOffset = needle.NeedleOffset(int64(n.TotalSize))
	if err = b.available(incrOffset); err != nil {
		return
	}
	if err = n.Write(b.bw); err != nil {
		b.LastErr = err
		return
	}
	b.Offset += incrOffset
	return
}

func compareTestOffset(b *SuperBlock, n *needle.Needle, offset uint32) (err error) {
	var v int64
	if b.Offset != offset+needle.NeedleOffset(int64(n.TotalSize)) {
		err = fmt.Errorf("b.Offset: %d not match", b.Offset)
		return
	}
	if v, err = b.w.Seek(0, os.SEEK_CUR); err != nil {
		err = fmt.Errorf("b.Seek() error(%v)", err)
		return
	} else {
		if v != needle.BlockOffset(b.Offset) {
			err = fmt.Errorf("offset: %d not match", v)
			return
		}
	}
	return
}

// Write write needle to the block.
func (b *SuperBlock) Write(data []byte) (err error) {
	var (
		size       = int64(len(data))
		incrOffset = needle.NeedleOffset(size)
	)
	if b.LastErr != nil {
		return b.LastErr
	}
	if _maxOffset-incrOffset < b.Offset {
		err = errors.ErrSuperBlockNoSpace
		return
	}
	if _, err = b.w.Write(data); err == nil {
		err = b.flush(false)
	} else {
		b.LastErr = err
		return
	}
	b.Offset += incrOffset
	b.Size += size
	return
}

func TestSuperBlock(t *testing.T) {
	var (
		b                  *SuperBlock
		offset, v2, v3, v4 uint32
		buf                []byte
		err                error
		n                  = &needle.Needle{}
		needles            = make(map[int64]int64)
		data               = []byte("test")
		file               = "../test/test.block"
		ifile              = "../test/test.idx"
		//indexer *Indexer
	)
	os.Remove(file)
	os.Remove(ifile)
	defer os.Remove(file)
	defer os.Remove(ifile)
	// test new block file
	if b, err = NewSuperBlock(file, Options{
		BufferSize:    4 * 1024 * 1024,
		SyncAtWrite:   1024,
		Syncfilerange: true,
	}); err != nil {
		t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
		t.FailNow()
	}
	b.Close()
	// test parse block file
	if b, err = NewSuperBlock(file, Options{
		BufferSize:    4 * 1024 * 1024,
		SyncAtWrite:   1024,
		Syncfilerange: true,
	}); err != nil {
		t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
		t.FailNow()
	}
	b.Close()
	// test open
	if err = b.Open(); err != nil {
		t.Errorf("Open() error(%v)", err)
		t.FailNow()
	}
	defer b.Close()
	// test add
	n.Parse(1, 1, data)
	if err = b.Add(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestOffset(b, n, needle.NeedleOffset(int64(headerSize))); err != nil {
		t.Errorf("compareTestOffset() error(%v)", err)
		t.FailNow()
	}
	offset = b.Offset
	v2 = b.Offset
	// test get
	buf = make([]byte, 40)
	if err = b.Get(1, buf); err != nil {
		t.Errorf("Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 1, 1, needle.FlagOK, n, data, buf); err != nil {
		t.Errorf("compareTestNeedle() error(%v)", err)
		t.FailNow()
	}
	// test add
	n.Parse(2, 2, data)
	if err = b.Add(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestOffset(b, n, offset); err != nil {
		t.Errorf("compareTestOffset() error(%v)", err)
		t.FailNow()
	}
	offset = b.Offset
	v3 = b.Offset
	if err = b.Get(6, buf); err != nil {
		t.Errorf("Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 2, 2, needle.FlagOK, n, data, buf); err != nil {
		t.Error("compareTestNeedle(2)")
		t.FailNow()
	}
	// test write
	n.Parse(3, 3, data)
	if err = b.Write(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	offset = b.Offset
	v4 = b.Offset
	// test write
	n.Parse(4, 4, data)
	if err = b.Write(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = b.Flush(); err != nil {
//.........这里部分代码省略.........

func TestIndex(t *testing.T) {
	var (
		i       *Indexer
		err     error
		noffset uint32
		file    = "../test/test.idx"
		needles = make(map[int64]int64)
	)
	os.Remove(file)
	defer os.Remove(file)
	if i, err = NewIndexer(file, Options{
		BufferSize:    4 * 1024 * 1024,
		MergeAtTime:   10 * time.Second,
		MergeAtWrite:  5,
		RingBuffer:    10,
		SyncAtWrite:   10,
		Syncfilerange: true,
		NeedleMaxSize: 4 * 1024 * 1024,
	}); err != nil {
		t.Errorf("NewIndexer() error(%v)", err)
		t.FailNow()
	}
	i.Close()
	// test closed
	if err = i.Add(1, 1, 8); err != errors.ErrIndexClosed {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	// test open
	if err = i.Open(); err != nil {
		t.Errorf("Open() error(%v)", err)
		t.FailNow()
	}
	defer i.Close()
	// test add
	if err = i.Add(1, 1, 8); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = i.Add(2, 2, 8); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = i.Add(5, 3, 8); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if err = i.Add(6, 4, 8); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	i.Signal()
	time.Sleep(1 * time.Second)
	i.Flush()
	// test recovery
	if err = i.Recovery(func(ix *Index) error {
		needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		noffset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		return nil
	}); err != nil {
		t.Errorf("Recovery() error(%v)", err)
		t.FailNow()
	}
	// add 4 index, start with 5
	if noffset != 5 {
		t.Errorf("noffset: %d not match", noffset)
		t.FailNow()
	}
	if o, s := needle.Cache(needles[1]); o != 1 && s != 8 {
		t.Error("needle cache not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[2]); o != 2 && s != 8 {
		t.Error("needle cache not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[5]); o != 3 && s != 8 {
		t.Error("needle cache not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[6]); o != 4 && s != 8 {
		t.Error("needle cache not match")
		t.FailNow()
	}
	// test write
	if err = i.Write(10, 5, 8); err != nil {
		t.Error("Write() error(%v)", err)
		t.FailNow()
	}
	if err = i.Flush(); err != nil {
		t.Error("Flush() error(%v)", err)
		t.FailNow()
	}
	// test recovery
	noffset = 0
	if err = i.Recovery(func(ix *Index) error {
		needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size)
		noffset = ix.Offset + needle.NeedleOffset(int64(ix.Size))
		return nil
	}); err != nil {
//.........这里部分代码省略.........

// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var (
		data   []byte
		so, eo uint32
		bso    int64
		fi     os.FileInfo
		fd     = r.Fd()
		n      = &needle.Needle{}
		rd     = bufio.NewReaderSize(r, b.Options.BufferSize)
	)
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	so, eo = offset, offset
	bso = needle.BlockOffset(so)
	// advise sequential read
	if fi, err = r.Stat(); err != nil {
		log.Errorf("block: %s Stat() error(%v)", b.File)
		return
	}
	if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	log.Infof("scan block: %s from offset: %d", b.File, offset)
	if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File)
		return
	}
	for {
		// header
		if data, err = rd.Peek(needle.HeaderSize); err != nil {
			break
		}
		if err = n.ParseHeader(data); err != nil {
			break
		}
		if _, err = rd.Discard(needle.HeaderSize); err != nil {
			break
		}
		// data
		if data, err = rd.Peek(int(n.Size)); err != nil {
			break
		}
		if err = n.ParseData(data); err != nil {
			break
		}
		if _, err = rd.Discard(int(n.Size)); err != nil {
			break
		}
		// footer
		if data, err = rd.Peek(int(n.FooterSize)); err != nil {
			break
		}
		if err = n.ParseFooter(data); err != nil {
			break
		}
		if _, err = rd.Discard(int(n.FooterSize)); err != nil {
			break
		}
		if log.V(1) {
			log.Info(n.String())
		}
		eo += needle.NeedleOffset(int64(n.TotalSize))
		if err = fn(n, so, eo); err != nil {
			break
		}
		so = eo
	}
	if err == io.EOF {
		// advise no need page cache
		if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
			log.Errorf("block: %s Fadvise() error(%v)", b.File)
			return
		}
		log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
		err = nil
	} else {
		log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
	}
	return
}

// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var (
		so, eo uint32
		bso    int64
		fi     os.FileInfo
		fd     = r.Fd()
		n      = needle.NewNeedle(b.Options.NeedleMaxSize)
		rd     = bufio.NewReaderSize(r, b.Options.BufferSize)
	)
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	so, eo = offset, offset
	bso = needle.BlockOffset(so)
	// advise sequential read
	if fi, err = r.Stat(); err != nil {
		log.Errorf("block: %s Stat() error(%v)", b.File)
		return
	}
	if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	log.Infof("scan block: %s from offset: %d", b.File, offset)
	if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File)
		return
	}
	for {
		if err = n.ParseFrom(rd); err != nil {
			if err != io.EOF {
				log.Errorf("block: parse needle from offset: %d:%d error(%v)", so, eo, err)
			}
			break
		}
		if n.Size > int32(b.Options.NeedleMaxSize) {
			log.Error("scan block: %s error(%v)", n, errors.ErrNeedleSize)
			err = errors.ErrNeedleSize
			break
		}
		if log.V(1) {
			log.Info(n.String())
		}
		eo += needle.NeedleOffset(int64(n.TotalSize))
		if err = fn(n, so, eo); err != nil {
			log.Errorf("block: callback from offset: %d:%d error(%v)", so, eo, err)
			break
		}
		so = eo
	}
	if err == io.EOF {
		// advise no need page cache
		if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
			log.Errorf("block: %s Fadvise() error(%v)", b.File)
			return
		}
		log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
		err = nil
	} else {
		log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
	}
	return
}